From 692dc294d60a72bfc5d59d72fc690e6f650d99f0 Mon Sep 17 00:00:00 2001 From: Anna Wiggins Date: Sat, 21 Apr 2012 04:27:05 -0400 Subject: [PATCH] Included gomill framework for SGF and GTP support, and sketched out SGF game-loading code. --- gomill/PKG-INFO | 47 + gomill/README.txt | 188 ++++ gomill/docs/_static/gomill.css_t | 43 + gomill/docs/_templates/genindex.html | 35 + gomill/docs/_templates/wholetoc.html | 2 + gomill/docs/allplayalls.rst | 120 +++ gomill/docs/ascii_boards.rst | 52 + gomill/docs/boards.rst | 114 +++ gomill/docs/cem_tuner.rst | 376 +++++++ gomill/docs/changes.rst | 93 ++ gomill/docs/common.rst | 74 ++ gomill/docs/competition_types.rst | 64 ++ gomill/docs/competitions.rst | 370 +++++++ gomill/docs/conf.py | 124 +++ gomill/docs/contact.rst | 13 + gomill/docs/errors.rst | 187 ++++ gomill/docs/example_scripts.rst | 104 ++ gomill/docs/glossary.rst | 116 +++ gomill/docs/gomill_package.rst | 79 ++ gomill/docs/gtp_extensions.rst | 133 +++ gomill/docs/handicap_layout.rst | 36 + gomill/docs/index.rst | 24 + gomill/docs/install.rst | 152 +++ gomill/docs/intro.rst | 97 ++ gomill/docs/library.rst | 32 + gomill/docs/library_overview.rst | 74 ++ gomill/docs/licence.rst | 25 + gomill/docs/mcts_tuner.rst | 646 +++++++++++++ gomill/docs/playoffs.rst | 214 ++++ gomill/docs/python-inv.txt | 12 + gomill/docs/results.rst | 106 ++ gomill/docs/ringmaster.rst | 71 ++ gomill/docs/ringmaster_cmdline.rst | 80 ++ gomill/docs/settings.rst | 534 ++++++++++ gomill/docs/sgf.rst | 914 ++++++++++++++++++ gomill/docs/tournament_results.rst | 401 ++++++++ gomill/examples/clop_example.ctl | 78 ++ gomill/examples/find_forfeits.py | 54 ++ gomill/examples/gomill-clop | 470 +++++++++ gomill/examples/gtp_stateful_player | 118 +++ gomill/examples/gtp_test_player | 144 +++ gomill/examples/kgs_proxy.py | 146 +++ gomill/examples/mogo_wrapper.py | 46 + gomill/examples/show_sgf.py | 72 ++ gomill/examples/split_sgf_collection.py | 54 ++ gomill/examples/twogtp | 75 ++ gomill/gomill/__init__.py | 1 + gomill/gomill/allplayalls.py | 257 +++++ gomill/gomill/ascii_boards.py | 80 ++ gomill/gomill/ascii_tables.py | 149 +++ gomill/gomill/boards.py | 248 +++++ gomill/gomill/cem_tuners.py | 509 ++++++++++ gomill/gomill/common.py | 92 ++ gomill/gomill/compact_tracebacks.py | 97 ++ gomill/gomill/competition_schedulers.py | 167 ++++ gomill/gomill/competitions.py | 513 ++++++++++ gomill/gomill/game_jobs.py | 411 ++++++++ gomill/gomill/gtp_controller.py | 823 ++++++++++++++++ gomill/gomill/gtp_engine.py | 532 ++++++++++ gomill/gomill/gtp_games.py | 816 ++++++++++++++++ gomill/gomill/gtp_proxy.py | 262 +++++ gomill/gomill/gtp_states.py | 651 +++++++++++++ gomill/gomill/handicap_layout.py | 54 ++ gomill/gomill/job_manager.py | 218 +++++ gomill/gomill/mcts_tuners.py | 848 ++++++++++++++++ gomill/gomill/playoffs.py | 179 ++++ gomill/gomill/ringmaster_command_line.py | 122 +++ gomill/gomill/ringmaster_presenters.py | 180 ++++ gomill/gomill/ringmasters.py | 769 +++++++++++++++ gomill/gomill/settings.py | 436 +++++++++ gomill/gomill/sgf.py | 806 +++++++++++++++ gomill/gomill/sgf_grammar.py | 513 ++++++++++ gomill/gomill/sgf_moves.py | 99 ++ gomill/gomill/sgf_properties.py | 730 ++++++++++++++ gomill/gomill/terminal_input.py | 83 ++ gomill/gomill/tournament_results.py | 311 ++++++ gomill/gomill/tournaments.py | 317 ++++++ gomill/gomill/utils.py | 74 ++ gomill/gomill_tests/__init__.py | 1 + gomill/gomill_tests/allplayall_tests.py | 289 ++++++ gomill/gomill_tests/board_test_data.py | 395 ++++++++ gomill/gomill_tests/board_tests.py | 184 ++++ gomill/gomill_tests/cem_tuner_tests.py | 227 +++++ gomill/gomill_tests/common_tests.py | 72 ++ .../competition_scheduler_tests.py | 98 ++ .../gomill_tests/competition_test_support.py | 80 ++ gomill/gomill_tests/competition_tests.py | 165 ++++ gomill/gomill_tests/game_job_tests.py | 458 +++++++++ gomill/gomill_tests/gomill_test_support.py | 161 +++ .../gtp_controller_test_support.py | 141 +++ gomill/gomill_tests/gtp_controller_tests.py | 698 +++++++++++++ gomill/gomill_tests/gtp_engine_fixtures.py | 390 ++++++++ .../gomill_tests/gtp_engine_test_support.py | 51 + gomill/gomill_tests/gtp_engine_tests.py | 56 ++ gomill/gomill_tests/gtp_game_tests.py | 630 ++++++++++++ gomill/gomill_tests/gtp_proxy_tests.py | 242 +++++ gomill/gomill_tests/gtp_state_test_support.py | 106 ++ gomill/gomill_tests/gtp_state_tests.py | 581 +++++++++++ gomill/gomill_tests/mcts_tuner_tests.py | 481 +++++++++ gomill/gomill_tests/playoff_tests.py | 570 +++++++++++ .../gomill_tests/ringmaster_test_support.py | 93 ++ gomill/gomill_tests/ringmaster_tests.py | 520 ++++++++++ gomill/gomill_tests/run_gomill_testsuite.py | 111 +++ gomill/gomill_tests/setting_tests.py | 29 + gomill/gomill_tests/sgf_grammar_tests.py | 362 +++++++ gomill/gomill_tests/sgf_moves_tests.py | 141 +++ gomill/gomill_tests/sgf_properties_tests.py | 386 ++++++++ gomill/gomill_tests/sgf_tests.py | 786 +++++++++++++++ .../gomill_tests/subprocess_state_reporter.py | 19 + gomill/gomill_tests/test_framework.py | 158 +++ gomill/gomill_tests/test_support.py | 78 ++ gomill/gomill_tests/utils_tests.py | 63 ++ gomill/ringmaster | 3 + gomill/setup.cfg | 7 + gomill/setup.py | 164 ++++ gomill/test_installed_gomill.py | 37 + lib/goban.py | 28 +- pygo.py | 16 +- ui/default.glade | 28 + 119 files changed, 27458 insertions(+), 3 deletions(-) create mode 100644 gomill/PKG-INFO create mode 100644 gomill/README.txt create mode 100644 gomill/docs/_static/gomill.css_t create mode 100644 gomill/docs/_templates/genindex.html create mode 100644 gomill/docs/_templates/wholetoc.html create mode 100644 gomill/docs/allplayalls.rst create mode 100644 gomill/docs/ascii_boards.rst create mode 100644 gomill/docs/boards.rst create mode 100644 gomill/docs/cem_tuner.rst create mode 100644 gomill/docs/changes.rst create mode 100644 gomill/docs/common.rst create mode 100644 gomill/docs/competition_types.rst create mode 100644 gomill/docs/competitions.rst create mode 100644 gomill/docs/conf.py create mode 100644 gomill/docs/contact.rst create mode 100644 gomill/docs/errors.rst create mode 100644 gomill/docs/example_scripts.rst create mode 100644 gomill/docs/glossary.rst create mode 100644 gomill/docs/gomill_package.rst create mode 100644 gomill/docs/gtp_extensions.rst create mode 100644 gomill/docs/handicap_layout.rst create mode 100644 gomill/docs/index.rst create mode 100644 gomill/docs/install.rst create mode 100644 gomill/docs/intro.rst create mode 100644 gomill/docs/library.rst create mode 100644 gomill/docs/library_overview.rst create mode 100644 gomill/docs/licence.rst create mode 100644 gomill/docs/mcts_tuner.rst create mode 100644 gomill/docs/playoffs.rst create mode 100644 gomill/docs/python-inv.txt create mode 100644 gomill/docs/results.rst create mode 100644 gomill/docs/ringmaster.rst create mode 100644 gomill/docs/ringmaster_cmdline.rst create mode 100644 gomill/docs/settings.rst create mode 100644 gomill/docs/sgf.rst create mode 100644 gomill/docs/tournament_results.rst create mode 100644 gomill/examples/clop_example.ctl create mode 100644 gomill/examples/find_forfeits.py create mode 100755 gomill/examples/gomill-clop create mode 100755 gomill/examples/gtp_stateful_player create mode 100755 gomill/examples/gtp_test_player create mode 100644 gomill/examples/kgs_proxy.py create mode 100644 gomill/examples/mogo_wrapper.py create mode 100644 gomill/examples/show_sgf.py create mode 100644 gomill/examples/split_sgf_collection.py create mode 100755 gomill/examples/twogtp create mode 100644 gomill/gomill/__init__.py create mode 100644 gomill/gomill/allplayalls.py create mode 100644 gomill/gomill/ascii_boards.py create mode 100644 gomill/gomill/ascii_tables.py create mode 100644 gomill/gomill/boards.py create mode 100644 gomill/gomill/cem_tuners.py create mode 100644 gomill/gomill/common.py create mode 100644 gomill/gomill/compact_tracebacks.py create mode 100644 gomill/gomill/competition_schedulers.py create mode 100644 gomill/gomill/competitions.py create mode 100644 gomill/gomill/game_jobs.py create mode 100644 gomill/gomill/gtp_controller.py create mode 100644 gomill/gomill/gtp_engine.py create mode 100644 gomill/gomill/gtp_games.py create mode 100644 gomill/gomill/gtp_proxy.py create mode 100644 gomill/gomill/gtp_states.py create mode 100644 gomill/gomill/handicap_layout.py create mode 100644 gomill/gomill/job_manager.py create mode 100644 gomill/gomill/mcts_tuners.py create mode 100644 gomill/gomill/playoffs.py create mode 100644 gomill/gomill/ringmaster_command_line.py create mode 100644 gomill/gomill/ringmaster_presenters.py create mode 100644 gomill/gomill/ringmasters.py create mode 100644 gomill/gomill/settings.py create mode 100644 gomill/gomill/sgf.py create mode 100644 gomill/gomill/sgf_grammar.py create mode 100644 gomill/gomill/sgf_moves.py create mode 100644 gomill/gomill/sgf_properties.py create mode 100644 gomill/gomill/terminal_input.py create mode 100644 gomill/gomill/tournament_results.py create mode 100644 gomill/gomill/tournaments.py create mode 100644 gomill/gomill/utils.py create mode 100644 gomill/gomill_tests/__init__.py create mode 100644 gomill/gomill_tests/allplayall_tests.py create mode 100644 gomill/gomill_tests/board_test_data.py create mode 100644 gomill/gomill_tests/board_tests.py create mode 100644 gomill/gomill_tests/cem_tuner_tests.py create mode 100644 gomill/gomill_tests/common_tests.py create mode 100644 gomill/gomill_tests/competition_scheduler_tests.py create mode 100644 gomill/gomill_tests/competition_test_support.py create mode 100644 gomill/gomill_tests/competition_tests.py create mode 100644 gomill/gomill_tests/game_job_tests.py create mode 100644 gomill/gomill_tests/gomill_test_support.py create mode 100644 gomill/gomill_tests/gtp_controller_test_support.py create mode 100644 gomill/gomill_tests/gtp_controller_tests.py create mode 100644 gomill/gomill_tests/gtp_engine_fixtures.py create mode 100644 gomill/gomill_tests/gtp_engine_test_support.py create mode 100644 gomill/gomill_tests/gtp_engine_tests.py create mode 100644 gomill/gomill_tests/gtp_game_tests.py create mode 100644 gomill/gomill_tests/gtp_proxy_tests.py create mode 100644 gomill/gomill_tests/gtp_state_test_support.py create mode 100644 gomill/gomill_tests/gtp_state_tests.py create mode 100644 gomill/gomill_tests/mcts_tuner_tests.py create mode 100644 gomill/gomill_tests/playoff_tests.py create mode 100644 gomill/gomill_tests/ringmaster_test_support.py create mode 100644 gomill/gomill_tests/ringmaster_tests.py create mode 100644 gomill/gomill_tests/run_gomill_testsuite.py create mode 100644 gomill/gomill_tests/setting_tests.py create mode 100644 gomill/gomill_tests/sgf_grammar_tests.py create mode 100644 gomill/gomill_tests/sgf_moves_tests.py create mode 100644 gomill/gomill_tests/sgf_properties_tests.py create mode 100644 gomill/gomill_tests/sgf_tests.py create mode 100644 gomill/gomill_tests/subprocess_state_reporter.py create mode 100644 gomill/gomill_tests/test_framework.py create mode 100644 gomill/gomill_tests/test_support.py create mode 100644 gomill/gomill_tests/utils_tests.py create mode 100755 gomill/ringmaster create mode 100644 gomill/setup.cfg create mode 100644 gomill/setup.py create mode 100644 gomill/test_installed_gomill.py diff --git a/gomill/PKG-INFO b/gomill/PKG-INFO new file mode 100644 index 0000000..740f444 --- /dev/null +++ b/gomill/PKG-INFO @@ -0,0 +1,47 @@ +Metadata-Version: 1.0 +Name: gomill +Version: 0.7.2 +Summary: Tools for testing and tuning Go-playing programs +Home-page: http://mjw.woodcraft.me.uk/gomill/ +Author: Matthew Woodcraft +Author-email: matthew@woodcraft.me.uk +License: MIT +Download-URL: http://mjw.woodcraft.me.uk/gomill/download/gomill-0.7.2.tar.gz +Description: Gomill is a suite of tools, and a Python library, for use in developing and + testing Go-playing programs. It is based around the Go Text Protocol (GTP) and + the Smart Game Format (SGF). + + The principal tool is the ringmaster, which plays programs against each other + and keeps track of the results. + + There is also experimental support for automatically tuning program parameters. + + Download: http://mjw.woodcraft.me.uk/gomill/download/gomill-0.7.2.tar.gz + + Documentation: http://mjw.woodcraft.me.uk/gomill/download/gomill-doc-0.7.2.tar.gz + + Online Documentation: http://mjw.woodcraft.me.uk/gomill/doc/0.7.2/ + + Changelog: http://mjw.woodcraft.me.uk/gomill/doc/0.7.2/changes.html + + Git: http://mjw.woodcraft.me.uk/gomill/git/ + + Gitweb: http://mjw.woodcraft.me.uk/gitweb/gomill/ + + +Keywords: go,baduk,weiqi,gtp,sgf +Platform: POSIX +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Natural Language :: English +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python +Classifier: Topic :: Games/Entertainment :: Board Games +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/gomill/README.txt b/gomill/README.txt new file mode 100644 index 0000000..9f59039 --- /dev/null +++ b/gomill/README.txt @@ -0,0 +1,188 @@ +Gomill +====== + +Gomill is a suite of tools, and a Python library, for use in developing and +testing Go-playing programs. + +Updated versions of Gomill will be made available at +http://mjw.woodcraft.me.uk/gomill/ + +The documentation is distributed separately in HTML form. It can be downloaded +from the above web site, or viewed online at +http://mjw.woodcraft.me.uk/gomill/doc/ + +A Git repository containing Gomill releases (but not detailed history) is +available: + git clone http://mjw.woodcraft.me.uk/gomill/git/ +It has a web interface at http://mjw.woodcraft.me.uk/gitweb/gomill/ + + +Contents +-------- + +The contents of the distribution directory (the directory containing this +README file) include: + + ringmaster -- Executable wrapper for the ringmaster program + gomill -- Python source for the gomill package + gomill_tests -- Test suite for the gomill package + docs -- ReST sources for the HTML documentation + examples -- Example scripts using the gomill library + setup.py -- Installation script + + +Requirements +------------ + +Gomill requires Python 2.5, 2.6, or 2.7. + +For Python 2.5 only, the --parallel feature requires the external +`multiprocessing` package [1]. + +Gomill is intended to run on any modern Unix-like system. + +[1] http://pypi.python.org/pypi/multiprocessing + + +Running the ringmaster +---------------------- + +The ringmaster executable in the distribution directory can be run directly +without any further installation; it will use the copy of the gomill package +in the distribution directory. + +A symbolic link to the ringmaster executable will also work, but if you move +the executable elsewhere it will not be able to find the gomill package unless +the package is installed. + + +Installation +------------ + +Installing Gomill puts the gomill package onto the Python module search path, +and the ringmaster executable onto the executable PATH. + +To install, first change to the distribution directory, then: + + - to install for the system as a whole, run (as a sufficiently privileged user) + + python setup.py install + + + - to install for the current user only (Python 2.6 or 2.7), run + + python setup.py install --user + + (in this case the ringmaster executable will be placed in ~/.local/bin.) + +Pass --dry-run to see what these will do. +See http://docs.python.org/2.7/install/ for more information. + + +Uninstallation +-------------- + +To remove an installed version of Gomill, run + + python setup.py uninstall + +(This uses the Python module search path and the executable PATH to find the +files to remove; pass --dry-run to see what it will do.) + + +Running the test suite +---------------------- + +To run the testsuite against the distributed gomill package, change to the +distribution directory and run + + python -m gomill_tests.run_gomill_testsuite + + +To run the testsuite against an installed gomill package, change to the +distribution directory and run + + python test_installed_gomill.py + + +With Python versions earlier than 2.7, the unittest2 library [1] is required +to run the testsuite. + +[1] http://pypi.python.org/pypi/unittest2/ + + +Running the example scripts +--------------------------- + +To run the example scripts, it is simplest to install the gomill package +first. + +If you do not wish to do so, you can run + + export PYTHONPATH= + +so that the example scripts will be able to find the gomill package. + + +Building the HTML documentation +------------------------------- + +To build the HTML documentation, change to the distribution directory and run + + python setup.py build_sphinx + +The documentation will be generated in build/sphinx/html. + +Requirements: + + Sphinx [1] version 1.0 or later (at least 1.0.4 recommended) + LaTeX [2] + dvipng [3] + +[1] http://sphinx.pocoo.org/ +[2] http://www.latex-project.org/ +[3] http://www.nongnu.org/dvipng/ + + +Licence +------- + +Gomill is copyright 2009-2011 Matthew Woodcraft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Contact +------- + +Please send any bug reports, suggestions, patches, questions &c to + +Matthew Woodcraft +matthew@woodcraft.me.uk + +I'm particularly interested in hearing about any GTP engines (even buggy ones) +which don't work with the ringmaster. + + +Changelog +--------- + +See the 'Changes' page in the HTML documentation (docs/changes.rst). + + mjw 2011-09-05 diff --git a/gomill/docs/_static/gomill.css_t b/gomill/docs/_static/gomill.css_t new file mode 100644 index 0000000..f373ccc --- /dev/null +++ b/gomill/docs/_static/gomill.css_t @@ -0,0 +1,43 @@ +@import url("default.css"); + +p.topic-title { + color: {{ theme_headtextcolor }}; +} + +dl.setting dd > p:first-child, +dl.mc-setting dd > p:first-child, +dl.ce-setting dd > p:first-child, +dl.gtp dd > p:first-child { + font-style: italic; +} + +tt.std-gtp span.pre { + white-space: nowrap; +} + +li.current > a {color: yellow;} + +div.tip { + background-color: #EEEEEE; + border: 1px solid #CCCCCC; +} + +div.caution { + background-color: #EEEEEE; + border: 1px solid #A00000; +} + +abbr { + border-bottom: none; cursor:help; +} + +th.field-name { + background-color: #E4E4E4; + font-weight: normal; +} + +div#library-overview table.docutils { + width : 100%; + margin-bottom: 3ex; +} + diff --git a/gomill/docs/_templates/genindex.html b/gomill/docs/_templates/genindex.html new file mode 100644 index 0000000..5aeb51b --- /dev/null +++ b/gomill/docs/_templates/genindex.html @@ -0,0 +1,35 @@ +{% extends "!genindex.html" %} +{% block body %} + +

{{ _('Index') }}

+ +
+ {% for key, dummy in genindexentries -%} + {{ key }} {% if not loop.last %}| {% endif %} + {%- endfor %} +
+ + {%- for key, entries in genindexentries %} +

{{ key }}

+ + +
+ {%- for entryname, (links, subitems) in entries %} +
{% if links %}{{ entryname|e }} + {%- for link in links[1:] %}, [{{ loop.index }}]{% endfor %} + {%- else %}{{ entryname|e }}{% endif %}
+ {%- if subitems %} +
+ {%- for subentryname, subentrylinks in subitems %} +
{{ subentryname|e }} + {%- for link in subentrylinks[1:] %}, [{{ loop.index }}]{% endfor -%} +
+ {%- endfor %} +
+ {%- endif -%} +{%- endfor %} +
+{% endfor %} + +{% endblock %} + diff --git a/gomill/docs/_templates/wholetoc.html b/gomill/docs/_templates/wholetoc.html new file mode 100644 index 0000000..e39bb63 --- /dev/null +++ b/gomill/docs/_templates/wholetoc.html @@ -0,0 +1,2 @@ +

{{ _('Table Of Contents') }}

+{{ toctree(collapse=False) }} diff --git a/gomill/docs/allplayalls.rst b/gomill/docs/allplayalls.rst new file mode 100644 index 0000000..66594a5 --- /dev/null +++ b/gomill/docs/allplayalls.rst @@ -0,0 +1,120 @@ +.. index:: all-play-all + +All-play-all tournaments +^^^^^^^^^^^^^^^^^^^^^^^^ + +:setting:`competition_type` string: ``"allplayall"``. + +In an all-play-all tournament the control file lists a number of players (the +:dfn:`competitors`), and games are played between each possible pairing. + +All games are played with no handicap and with the same komi. The players in +each pairing will swap colours in successive games. + +For most purposes an all-play-all tournament is equivalent to a playoff +tournament with a matchup defined for each pair of competitors; the main +difference is that reports include a results summary grid. + +The tournament runs until :aa-setting:`rounds` games have been played between +each pairing (indefinitely, if :aa-setting:`rounds` is unset). + + +.. contents:: Page contents + :local: + :backlinks: none + + +.. _sample_allplayall_control_file: + +Sample control file +""""""""""""""""""" + +Here is a sample control file:: + + competition_type = 'allplayall' + + players = { + 'gnugo-l1' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=1"), + + 'gnugo-l2' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=2"), + + 'gnugo-l3' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=3"), + } + + board_size = 9 + komi = 6 + + rounds = 20 + competitors = ['gnugo-l1', 'gnugo-l2', 'gnugo-l3'] + + +.. _allplayall_control_file_settings: + +Control file settings +""""""""""""""""""""" + +The following settings can be set at the top level of the control file: + +All :ref:`common settings `. + +The following game settings: :setting:`board_size`, :setting:`komi`, +:setting:`move_limit`, :setting:`scorer`. + +The following additional settings: + +.. aa-setting:: competitors + + List of :ref:`player codes `. + + This defines which players will take part. Reports will list the players + in the order in which they appear here. You may not list the same player + more than once. + +.. aa-setting:: rounds + + Integer (default ``None``) + + The number of games to play for each pairing. If you leave this unset, the + tournament will continue indefinitely. + +The only required settings are :setting:`competition_type`, +:setting:`players`, :aa-setting:`competitors`, :setting:`board_size`, and +:setting:`komi`. + + +Reporting +""""""""" + +The :ref:`live display ` and :ref:`competition report +` summarise the tournament results in the form of a +grid, for example:: + + A B C + A gnugo-l1 4-5 3-5 + B gnugo-l2 5-4 3-5 + C gnugo-l3 5-3 5-3 + +Each row shows the number of wins and losses for the player named on that row +against each opponent (in the example, ``gnugo-l1`` has won 4 games and lost 5 +against ``gnugo-l2``). + +If any games have unknown results (because they could not be scored, or +reached the :setting:`move_limit`), they will not be shown in the grid. + +The competition report also shows full details of each pairing in the same +style as playoff tournaments. + +For purposes of the :doc:`tournament results API `, the +matchup ids are of the form ``AvB`` (using the competitor letters shown in the +results grid). + + +Changing the control file between runs +"""""""""""""""""""""""""""""""""""""" + +You can add new players to the end of the :aa-setting:`competitors` list +between runs, but you may not remove or reorder competitors. + diff --git a/gomill/docs/ascii_boards.rst b/gomill/docs/ascii_boards.rst new file mode 100644 index 0000000..99acd6c --- /dev/null +++ b/gomill/docs/ascii_boards.rst @@ -0,0 +1,52 @@ +The :mod:`~gomill.ascii_boards` module +-------------------------------------- + +.. module:: gomill.ascii_boards + :synopsis: ASCII Go board diagrams. + +The :mod:`!gomill.ascii_boards` module contains functions for producing and +interpreting ASCII diagrams of Go board positions. + + +.. function:: render_board(board) + + :rtype: string + + Returns an ASCII diagram of the position on the :class:`.Board` *board*. + + The returned string does not end with a newline. + + :: + + >>> b = boards.Board(9) + >>> b.play(2, 5, 'b') + >>> b.play(3, 6, 'w') + >>> print ascii_boards.render_board(b) + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . o . . + 3 . . . . . # . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J + + See also the :script:`show_sgf.py` example script. + + +.. function:: interpret_diagram(diagram, size[, board]) + + :rtype: :class:`.Board` + + Returns the position given in an ASCII diagram. + + *diagram* must be a string in the format returned by :func:`render_board`, + representing a position with the specified size. + + Raises :exc:`ValueError` if it can't interpret the diagram. + + If the optional *board* parameter is provided, it must be an empty + :class:`.Board` of the right size; the same object will be returned (this + option is provided so you can use a different Board class). diff --git a/gomill/docs/boards.rst b/gomill/docs/boards.rst new file mode 100644 index 0000000..f09f5ea --- /dev/null +++ b/gomill/docs/boards.rst @@ -0,0 +1,114 @@ +The :mod:`~gomill.boards` module +-------------------------------- + +.. module:: gomill.boards + :synopsis: Go board representation. + +The :mod:`!gomill.boards` module contains Gomill's Go board representation. + +Everything in this module works with boards of arbitrarily large sizes. + +The implementation is not designed for speed (even as Python code goes), and +is certainly not appropriate for implementing a playing engine. + +The module contains a single class: + + +.. class:: Board(side) + + A :class:`!Board` object represents a legal position on a Go board. + + Instantiate with the board size, as an int >= 1. Only square boards are + supported. The board is initially empty. + + Board objects do not maintain any history information. + + Board objects have the following attributes (which should be treated as + read-only): + + .. attribute:: side + + The board size. + + .. attribute:: board_points + + A list of *points*, giving all points on the board. + + +The principal :class:`!Board` methods are :meth:`!get` and :meth:`!play`. +Their *row* and *col* parameters should be ints representing coordinates in +the :ref:`system ` used for a *point*. + +The behaviour of :class:`!Board` methods is unspecified if they are passed +out-of-range coordinates. + +.. method:: Board.get(row, col) + + :rtype: *colour* or ``None`` + + Returns the contents of the specified point. + +.. method:: Board.play(row, col, colour) + + :rtype: *move* + + Places a stone of the specified *colour* on the specified point. + + Raises :exc:`ValueError` if the point isn't empty. + + Carries out any captures which follow from the placement, including + self-captures. + + This method doesn't enforce any ko rule. + + The return value indicates whether, immediately following this move, any + point would be forbidden by the :term:`simple ko` rule. If so, that point + is returned; otherwise the return value is ``None``. + + +The other :class:`!Board` methods are: + +.. method:: Board.is_empty() + + :rtype: bool + + Returns ``True`` if all points on the board are empty. + +.. method:: Board.list_occupied_points() + + :rtype: list of pairs (*colour*, *point*) + + Returns a list of all nonempty points, in unspecified order. + +.. method:: Board.area_score() + + :rtype: int + + Calculates the area score of a position, assuming that all stones are + alive. The result is the number of points controlled (occupied or + surrounded) by Black minus the number of points controlled by White. + + Doesn't take any :term:`komi` into account. + +.. method:: Board.copy() + + :rtype: :class:`!Board` + + Returns an independent copy of the board. + +.. method:: Board.apply_setup(black_points, white_points, empty_points) + + :rtype: bool + + Adds and/or removes stones on arbitrary points. This is intended to support + behaviour like |sgf| ``AB``/``AW``/``AE`` properties. + + Each parameter is an iterable of *points*. + + This method applies all the specified additions and removals, then removes + any groups with no liberties (so the resulting position is always legal). + + If the same point is specified in more than one list, the order in which + the instructions are applied is undefined. + + Returns ``True`` if the position was legal as specified. diff --git a/gomill/docs/cem_tuner.rst b/gomill/docs/cem_tuner.rst new file mode 100644 index 0000000..5cfde71 --- /dev/null +++ b/gomill/docs/cem_tuner.rst @@ -0,0 +1,376 @@ +.. |ce| replace:: :ref:`[CE] ` + +The cross-entropy tuner +^^^^^^^^^^^^^^^^^^^^^^^ + +:setting:`competition_type` string: ``"ce_tuner"``. + +The cross-entropy tuner uses the :dfn:`cross-entropy method` described in +|ce|: + +.. _ce_paper: + +| [CE] G.M.J-B. Chaslot, M.H.M Winands, I. Szita, and H.J. van den Herik. +| Cross-entropy for Monte-Carlo Tree Search. ICGA Journal, 31(3):145-156. +| http://www.personeel.unimaas.nl/g-chaslot/papers/crossmcICGA.pdf + +.. caution:: The cross-entropy tuner is experimental. It can take a very large + number of games to converge. + + +.. contents:: Page contents + :local: + :backlinks: none + + +The tuning algorithm +"""""""""""""""""""" + +The algorithm is not described in detail in this documentation. See |ce| +section 3 for the description. The tuner always uses a Gaussian distribution. +The improvement suggested in section 5 is not implemented. + + +.. _ce parameter model: + +The parameter model +""""""""""""""""""" + +The parameter values taken from the Gaussian distribution are floating-point +numbers known as :dfn:`optimiser parameters`. + +These parameters can be transformed before being used to configure the +candidate (see 3.3 *Normalising Parameters* in |ce|). The transformed values +are known as :dfn:`engine parameters`. The transformation is implemented using +a Python :ce-setting:`transform` function defined in the control file. + +Reports show engine parameters (see the :ce-setting:`format` parameter +setting), together with the mean and variance of the corresponding optimiser +parameter distribution in the form :samp:`{mean}~{variance}`. + + +.. _the cem tuning algorithm: + +.. _sample_cem_control_file: + +Sample control file +""""""""""""""""""" + +Here is a sample control file, illustrating most of the available settings for +a cross-entropy tuning event:: + + competition_type = "ce_tuner" + + description = """\ + This is a sample control file. + + It illustrates the available settings for the cross entropy tuner. + """ + + players = { + 'gnugo-l10' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=10"), + } + + def fuego(max_games, additional_commands=[]): + commands = [ + "go_param timelimit 999999", + "uct_max_memory 350000000", + "uct_param_search number_threads 1", + "uct_param_player reuse_subtree 0", + "uct_param_player ponder 0", + "uct_param_player max_games %d" % max_games, + ] + return Player( + "fuego --quiet", + startup_gtp_commands=commands+additional_commands) + + FUEGO_MAX_GAMES = 1000 + + def exp_10(f): + return 10.0**f + + parameters = [ + Parameter('rave_weight_initial', + # Mean and variance are in terms of log_10 (rave_weight_initial) + initial_mean = -1.0, + initial_variance = 1.5, + transform = exp_10, + format = "I: %4.2f"), + + Parameter('rave_weight_final', + # Mean and variance are in terms of log_10 (rave_weight_final) + initial_mean = 3.5, + initial_variance = 1.5, + transform = exp_10, + format = "F: %4.2f"), + ] + + def make_candidate(rwi, rwf): + return fuego( + FUEGO_MAX_GAMES, + ["uct_param_search rave_weight_initial %f" % rwi, + "uct_param_search rave_weight_final %f" % rwf]) + + board_size = 9 + komi = 7.5 + opponent = 'gnugo-l10' + candidate_colour = 'w' + + number_of_generations = 5 + samples_per_generation = 100 + batch_size = 10 + elite_proportion = 0.1 + step_size = 0.8 + + + +.. _cem_control_file_settings: + +Control file settings +""""""""""""""""""""" + +The following settings can be set at the top level of the control file: + +All :ref:`common settings ` (the :setting:`players` +dictionary is required, though it is used only to define the opponent). + +The following game settings (only :setting:`!board_size` and :setting:`!komi` +are required): + +- :setting:`board_size` +- :setting:`komi` +- :setting:`handicap` +- :setting:`handicap_style` +- :setting:`move_limit` +- :setting:`scorer` + + +The following additional settings (they are all required): + +.. ce-setting:: candidate_colour + + String: ``"b"`` or ``"w"`` + + The colour for the candidates to take in every game. + + +.. ce-setting:: opponent + + Identifier + + The :ref:`player code ` of the player to use as the + candidates' opponent. + + +.. ce-setting:: parameters + + List of :ce-setting-cls:`Parameter` definitions (see :ref:`ce parameter + configuration`). + + Describes the parameters that the tuner will work with. See :ref:`ce + parameter model` for more details. + + The order of the :ce-setting-cls:`Parameter` definitions is used for the + arguments to :ce-setting:`make_candidate`, and whenever parameters are + described in reports or game records. + + +.. ce-setting:: make_candidate + + Python function + + Function to create a :setting-cls:`Player` from its engine parameters. + + This function is passed one argument for each candidate parameter, and must + return a :setting-cls:`Player` definition. Each argument is the output of + the corresponding Parameter's :ce-setting:`transform`. + + The function will typically use its arguments to construct command line + options or |gtp| commands for the player. For example:: + + def make_candidate(param1, param2): + return Player(["goplayer", "--param1", str(param1), + "--param2", str(param2)]) + + def make_candidate(param1, param2): + return Player("goplayer", startup_gtp_commands=[ + ["param1", str(param1)], + ["param2", str(param2)], + ]) + + +.. ce-setting:: number_of_generations + + Positive integer + + The number of times to repeat the tuning algorithm (*number of iterations* + or *T* in the terminology of |ce|). + + +.. ce-setting:: samples_per_generation + + Positive integer + + The number of candidates to make in each generation (*population_size* or + *N* in the terminology of |ce|). + + +.. ce-setting:: batch_size + + Positive integer + + The number of games played by each candidate. + + +.. ce-setting:: elite_proportion + + Float between 0.0 and 1.0 + + The proportion of candidates to select from each generation as 'elite' (the + *selection ratio* or *ρ* in the terminology of |ce|). A value between 0.01 + and 0.1 is recommended. + + + +.. ce-setting:: step_size + + Float between 0.0 and 1.0 + + The rate at which to update the distribution parameters between generations + (*α* in the terminology of |ce|). + + .. caution:: I can't find anywhere in the paper the value they used for + this, so I don't know what to recommend. + + +.. _ce parameter configuration: + +Parameter configuration +""""""""""""""""""""""" + +.. ce-setting-cls:: Parameter + +A :ce-setting-cls:`!Parameter` definition has the same syntax as a Python +function call: :samp:`Parameter({arguments})`. Apart from :ce-setting:`!code`, +the arguments should be specified using keyword form (see +:ref:`sample_cem_control_file`). + +The :ce-setting:`code`, :ce-setting:`initial_mean`, and +:ce-setting:`initial_variance` arguments are required. + +The arguments are: + + +.. ce-setting:: code + + Identifier + + A short string used to identify the parameter. This is used in error + messages, and in the default for :ce-setting:`format`. + + +.. ce-setting:: initial_mean + + Float + + The mean value for the parameter in the first generation's distribution. + + +.. ce-setting:: initial_variance + + Float >= 0 + + The variance for the parameter in the first generation's distribution. + + +.. ce-setting:: transform + + Python function (default identity) + + Function mapping an optimiser parameter to an engine parameter; see :ref:`ce + parameter model`. + + Examples:: + + def exp_10(f): + return 10.0**f + + Parameter('p1', initial_mean = …, initial_variance = …, + transform = exp_10) + + If the :ce-setting:`!transform` is not specified, the optimiser parameter is + used directly as the engine parameter. + + +.. ce-setting:: format + + String (default :samp:`"{parameter_code}: %s"`) + + Format string used to display the parameter value. This should include a + short abbreviation to indicate which parameter is being displayed, and also + contain ``%s``, which will be replaced with the engine parameter value. + + You can use any Python conversion specifier instead of ``%s``. For example, + ``%.2f`` will format a floating point number to two decimal places. ``%s`` + should be safe to use for all types of value. See `string formatting + operations`__ for details. + + .. __: http://docs.python.org/release/2.7/library/stdtypes.html#string-formatting-operations + + Format strings should be kept short, as screen space is limited. + + Examples:: + + Parameter('parameter_1', + initial_mean = 0.0, initial_variance = 1.0, + format = "p1: %.2f") + + Parameter('parameter_2', + initial_mean = 5000, initial_variance = 250000, + format = "p2: %d") + + +Reporting +""""""""" + +Currently, there aren't any sophisticated reports. + +The standard report shows the parameters of the current Gaussian distribution, +and the number of wins for each candidate in the current generation. + +After each generation, the details of the candidates are written to the +:ref:`history file `. The candidates selected as elite are marked +with a ``*``. + + +Changing the control file between runs +"""""""""""""""""""""""""""""""""""""" + +Some settings can safely be changed between runs of the same cross-entropy +tuning event: + +:ce-setting:`batch_size` + safe to increase + +:ce-setting:`samples_per_generation` + not safe to change + +:ce-setting:`number_of_generations` + safe to change + +:ce-setting:`elite_proportion` + safe to change + +:ce-setting:`step_size` + safe to change + +:ce-setting:`make_candidate` + safe to change, but don't alter play-affecting options + +:ce-setting:`transform` + not safe to change + +:ce-setting:`format` + safe to change + diff --git a/gomill/docs/changes.rst b/gomill/docs/changes.rst new file mode 100644 index 0000000..9dacbce --- /dev/null +++ b/gomill/docs/changes.rst @@ -0,0 +1,93 @@ +Changes +======= + +Gomill 0.7.2 (2011-09-05) +------------------------- + +* Added the *wrap* parameter to :meth:`.Sgf_game.serialise`. + +* Added the :script:`gomill-clop` example script. + + +Gomill 0.7.1 (2011-08-15) +------------------------- + +Bug-fix release. + +* Bug fix: made board sizes 24 and 25 work (column lettering, and therefore + |gtp| support, was incorrect for these sizes in all previous versions). + +* Tightened up input validation for :func:`.format_vertex` and + :func:`.colour_name`. + +* Distinguished Stone, Point, and Move in the :ref:`sgf_property_types` + table in |sgf| documentation. + + + +Gomill 0.7 (2011-08-13) +----------------------- + +The ringmaster now applies handicap stone compensation when using its internal +scorer. Set :setting:`internal_scorer_handicap_compensation` to ``"no"`` to +return to the old behaviour. + +* Added a full implementation of :doc:`sgf`, replacing the previous minimal + support. + +* Added a :script:`split_sgf_collection.py` example script. + +* The :mod:`~gomill.common`, :mod:`~gomill.boards`, + :mod:`~gomill.ascii_boards`, and :mod:`~gomill.handicap_layout` modules are + now documented as stable. + +* Improved handling of long responses to the :gtp:`!version` |gtp| command. + +* Added support for handicap stone compensation when scoring games. + +* Gomill now checks the response to the :gtp:`!fixed_handicap` |gtp| command. + +* Added the :data:`gomill.__version__` constant. + + +Changes to (previously) undocumented parts of the library: + +* Renamed the :mod:`!gomill.gomill_common` module to :mod:`!gomill.common`. + +* Renamed the :mod:`!gomill.gomill_utils` module to :mod:`!gomill.utils`. + +* Renamed :attr:`!Board.board_coords` to :attr:`~.Board.board_points`. + +* Replaced the :func:`!ascii_boards.play_diagram` function with + :func:`~.ascii_boards.interpret_diagram`, making the *board* parameter + optional. + +* :func:`!gtp_engine.interpret_float` now rejects infinities and NaNs. + +* Changes to the :mod:`!gtp_states` module: tightened error handling, removed + the komi-mangling feature, renamed :attr:`!History_move.coords` to + :attr:`!History_move.move`. + + +Gomill 0.6 (2011-02-13) +----------------------- + +Playoff tournament :ref:`state files ` from Gomill 0.5 are +incompatible with Gomill 0.6. Tuning event state files are compatible. + +* Added the :doc:`All-play-all ` tournament type. + +* Expanded and documented the :doc:`tournament_results`. Changed return type + of + :meth:`~.Tournament_results.get_matchup_results`. + +* Fixed reporting for matchups with the same player specified twice. + +* Allowed arbitrary filename extensions for control files. + + +Gomill 0.5 (2010-10-29) +----------------------- + +* First public release. + diff --git a/gomill/docs/common.rst b/gomill/docs/common.rst new file mode 100644 index 0000000..94d41c7 --- /dev/null +++ b/gomill/docs/common.rst @@ -0,0 +1,74 @@ +The :mod:`~gomill.common` module +-------------------------------- + +.. module:: gomill.common + :synopsis: Go-related utility functions. + +The :mod:`!gomill.common` module provides Go-related utility functions, used +throughout Gomill. + +It is designed to be safe to use as ``from common import *``. + +.. function:: opponent_of(colour) + + :rtype: *colour* + + Returns the other colour:: + + >>> opponent_of('b') + 'w' + +.. function:: colour_name(colour) + + :rtype: string + + Returns the (lower-case) full name of a *colour*:: + + >>> colour_name('b') + 'black' + +.. function:: format_vertex(move) + + :rtype: string + + Returns a string describing a *move* in conventional notation:: + + >>> format_vertex((3, 0)) + 'A4' + >>> format_vertex(None) + 'pass' + + The result is suitable for use directly in |GTP| responses. Note that ``I`` + is omitted from the letters used to indicate columns, so the maximum + supported column value is ``25``. + +.. function:: format_vertex_list(moves) + + :rtype: string + + Returns a string describing a sequence of *moves*:: + + >>> format_vertex_list([(0, 1), (2, 3), None]) + 'B1,D3,pass' + >>> format_vertex_list([]) + '' + +.. function:: move_from_vertex(vertex, board_size) + + :rtype: *move* + + Interprets the string *vertex* as conventional notation, assuming a square + board whose side is *board_size*:: + + >>> move_from_vertex("A4", 9) + (3, 0) + >>> move_from_vertex("a4", 9) + (3, 0) + >>> move_from_vertex("pass", 9) + None + + Raises :exc:`ValueError` if it can't parse the string, or if the resulting + point would be off the board. + + Treats *vertex* case-insensitively. + diff --git a/gomill/docs/competition_types.rst b/gomill/docs/competition_types.rst new file mode 100644 index 0000000..1961fa3 --- /dev/null +++ b/gomill/docs/competition_types.rst @@ -0,0 +1,64 @@ +.. index:: competition type + +.. _competition types: + +Competition types +----------------- + +The ringmaster supports a number of different :dfn:`competition types`. These +are divided into :dfn:`tournaments` and :dfn:`tuning events`. + + +.. contents:: Page contents + :local: + :backlinks: none + + +.. index:: tournament + +.. _tournaments: + +Tournaments +^^^^^^^^^^^ + +A :dfn:`tournament` is a form of competition in which the ringmaster plays +games between predefined players, in order to compare their strengths. + + +There are currently two types of tournament: + +.. toctree:: + :maxdepth: 3 + :titlesonly: + + Playoff + All-play-all + + +.. index:: tuning event + +.. _tuners: + +Tuning events +^^^^^^^^^^^^^ + +A :dfn:`tuning event` is a form of competition in which the ringmaster runs an +algorithm which adjusts engine parameters to try to find the values which give +strongest play. + +.. index:: opponent + +At present, all tuning events work by playing games between different +:dfn:`candidate` players and a single fixed :dfn:`opponent` player. The +candidate always takes the same colour. The komi and any handicap can be +specified as usual. + +There are currently two tuning algorithms: + +.. toctree:: + :maxdepth: 3 + :titlesonly: + + Monte Carlo + Cross-entropy + diff --git a/gomill/docs/competitions.rst b/gomill/docs/competitions.rst new file mode 100644 index 0000000..3c53954 --- /dev/null +++ b/gomill/docs/competitions.rst @@ -0,0 +1,370 @@ +.. _running competitions: + +Running competitions +-------------------- + +.. contents:: Page contents + :local: + :backlinks: none + + +Pairings +^^^^^^^^ + +When a competition is run, the ringmaster will launch one or more games +between pairs of players. + +For playoff tournaments, the pairings are determined by the +:pl-setting-cls:`Matchup` descriptions in the control file. If more than one +matchup is specified, the ringmaster prefers to start games from the matchup +which has played fewest games. + +For all-play-all tournaments, the ringmaster will again prefer the pair of +:aa-setting:`competitors` which has played the fewest games. + +For tuning events, the pairings are specified by a tuning algorithm. + + +.. _simultaneous games: + +Simultaneous games +^^^^^^^^^^^^^^^^^^ + +The ringmaster can run more than one game at a time, if the +:option:`--parallel ` command line option is specified. + +This can be useful to keep processor cores busy, or if the actual playing +programs are running on different machines to the ringmaster. + +Normally it makes no difference whether the ringmaster starts games in +sequence or in parallel, but it does have an effect on the :doc:`Monte Carlo +tuner `, as in parallel mode it will have less information each +time it chooses a candidate player. + +.. tip:: Even if an engine is capable of using multiple threads, it may be + better to use a single-threaded configuration during development to get + reproducible results, or to be sure that system load does not affect play. + +.. tip:: When deciding how many games to run in parallel, remember to take + into account the amount of memory needed, as well as the number of + processor cores available. + + +.. _live_display: + +Display +^^^^^^^ + +While the competition runs, the ringmaster displays a summary of the +tournament results (or of the tuning algorithm status), a list of games in +progress, and a list of recent game results. For example, in a playoff +tournament with a single matchup:: + + 2 games in progress: 0_2 0_4 + (Ctrl-X to halt gracefully) + + gnugo-l1 v gnugo-l2 (3/5 games) + board size: 9 komi: 7.5 + wins black white avg cpu + gnugo-l1 2 66.67% 1 100.00% 1 50.00% 1.13 + gnugo-l2 1 33.33% 1 50.00% 0 0.00% 1.32 + 2 66.67% 1 33.33% + + = Results = + game 0_1: gnugo-l2 beat gnugo-l1 B+8.5 + game 0_0: gnugo-l1 beat gnugo-l2 B+33.5 + game 0_3: gnugo-l1 beat gnugo-l2 W+2.5 + +Use :ref:`quiet mode ` to turn this display off. + + +.. _stopping competitions: + +Stopping competitions +^^^^^^^^^^^^^^^^^^^^^ + +Unless interrupted, a run will continue until either the competition completes +or the per-run limit specified by the :option:`--max-games +` command line option is reached. + +Type :kbd:`Ctrl-X` to stop a run. The ringmaster will wait for all games in +progress to complete, and then exit (the stop request won't be acknowledged on +screen until the next game result comes in). + +It's also reasonable to stop a run with :kbd:`Ctrl-C`; games in progress will +be terminated immediately (assuming the engine processes are well-behaved). +The partial games will be forgotten; the ringmaster will replay them as +necessary if the competition is resumed later. + +You can also stop a competition by running the command line :action:`stop` +action from a shell; like :kbd:`Ctrl-X`, this will be acknowledged when the +next game result comes in, and the ringmaster will wait for games in progress +to complete. + + +Running players +^^^^^^^^^^^^^^^ + +The ringmaster requires the player engines to be standalone executables which +speak :term:`GTP` version 2 on their standard input and output streams. + +It launches the executables itself, with command line arguments and other +environment as detailed by the :ref:`player settings ` +in the control file. + +It launches a new engine subprocess for each game and closes it when the game +is terminated. + +.. tip:: To run a player on a different computer to the ringmaster, specify a + suitable :program:`ssh` command line in the :setting-cls:`Player` + definition. + +See :ref:`engine errors` and :ref:`engine exit behaviour` for details of what +happens if engines misbehave. + + +.. index:: rules, ko, superko + +.. _playing games: + +Playing games +^^^^^^^^^^^^^ + +The :setting:`board_size`, :setting:`komi`, :setting:`handicap`, and +:setting:`handicap_style` game settings control the details of the game. The +ringmaster doesn't know or care what rule variant the players are using; it's +up to you to make sure they agree with each other. + +Any :setting:`startup_gtp_commands` configured for a player will be sent +before the :gtp:`!boardsize` and :gtp:`!clear_board` commands. Failure responses +from these commands are ignored. + +Each game normally continues until both players pass in succession, or one +player resigns. + +The ringmaster rejects moves to occupied points, and moves forbidden by +:term:`simple ko`, as illegal. It doesn't reject self-capture moves, and it +doesn't enforce any kind of :term:`superko` rule. If the ringmaster rejects a +move, the player that tried to make it loses the game by forfeit. + +If one of the players rejects a move as illegal (ie, with the |gtp| failure +response ``illegal move``), the ringmaster assumes its opponent really has +played an illegal move and so should forfeit the game (this is convenient if +you're testing an experimental engine against an established one). + +If one of the players returns any other |gtp| failure response (either to +:gtp:`!genmove` or to :gtp:`!play`), or an uninterpretable response to +:gtp:`!genmove`, it forfeits the game. + +If the game lasts longer than the configured :setting:`move_limit`, it is +stopped at that point, and recorded as having an unknown result (with |sgf| +result ``Void``). + +See also :ref:`claiming wins`. + +.. note:: The ringmaster does not provide a game clock, and it does not + use any of the |gtp| time handling commands. Players should normally be + configured to use a fixed amount of computing power, independent of + wall-clock time. + + +.. index:: handicap compensation + +.. _scoring: + +Scoring +^^^^^^^ + +The ringmaster has two scoring methods: ``players`` (which is the default), +and ``internal``. The :setting:`scorer` game setting determines which is used. + +When the ``players`` method is used, the players are asked to score the game +using the |gtp| :gtp:`!final_score` command. See also the +:setting:`is_reliable_scorer` player setting. + +When the ``internal`` method is used, the ringmaster scores the game itself, +area-fashion. It assumes that all stones remaining on the board at the end of +the game are alive. It applies :setting:`komi`. + +In handicap games, the internal scorer can also apply handicap stone +compensation, controlled by the +:setting:`internal_scorer_handicap_compensation` game setting: ``"full"`` (the +default) means that White is given an additional point for each handicap +stone, ``"short"`` means White is given an additional point for each handicap +stone except the first, and ``"no"`` means that no handicap stone compensation +is given. + + +.. _claiming wins: + +Claiming wins +^^^^^^^^^^^^^ + +The ringmaster supports a protocol to allow players to declare that they have +won the game. This can save time if you're testing against opponents which +don't resign. + +To support this, the player has to implement :gtp:`gomill-genmove_ex` and +recognise the ``claim`` keyword. + +You must also set :setting:`allow_claim` ``True`` in the :setting-cls:`Player` +definition for this mechanism to be used. + +The |sgf| result of a claimed game will simply be ``B+`` or ``W+``. + + +.. _startup checks: + +Startup checks +^^^^^^^^^^^^^^ + +Whenever the ringmaster starts a run, before starting any games, it launches +an instance of each engine that will be required for the run and checks that +it operates reasonably. + +If any engine fails the checks, the run is cancelled. The standard error +stream from the engines is suppressed for these automatic startup checks. + +The :action:`check` command line action runs the same checks, but it leaves +the engines' standard error going to the console (any +:setting:`discard_stderr` player settings are ignored). + +For playoff tournaments, only players listed in matchups are checked (and +matchups with :pl-setting:`number_of_games` set to ``0`` are ignored). If a +player appears in more than one matchup, the board size and komi from its +first matchup are used. + +For all-play-all tournaments, all players listed as :aa-setting:`competitors` +are checked. + +For tuning events, the opponent and one sample candidate are checked. + +The checks are as follows: + +- the engine subprocess starts, and replies to |gtp| commands +- the engine reports |gtp| protocol version 2 (if it supports + :gtp:`!protocol_version` at all) +- the engine accepts any :setting:`startup_gtp_commands` +- the engine accepts the required board size and komi +- the engine accepts the :gtp:`!clear_board` |gtp| command + + +.. _quiet mode: + +.. index:: quiet mode + +Quiet mode +^^^^^^^^^^ + +The :option:`--quiet ` command line option makes the +ringmaster run in :dfn:`quiet mode`. In this mode, it prints nothing to +standard output, and only errors and warnings to standard error. + +This mode is suitable for running in the background. + +:kbd:`Ctrl-X` still works in quiet mode to stop a run gracefully, if the +ringmaster process is in the foreground. + + +.. _output files: + +.. _competition directory: + +Output files +^^^^^^^^^^^^ + +.. index:: competition directory + +The ringmaster writes a number of files, which it places in the directory +which contains the control file (the :dfn:`competition directory`). The +filename stem (the part before the filename extension) of each file is the +same as in the control file (:file:`{code}` in the table below). + +The full set of files that may be present in the competition directory is: + +======================= ======================================================= +:file:`{code}.ctl` the :doc:`control file ` +:file:`{code}.status` the :ref:`competition state ` file +:file:`{code}.log` the :ref:`event log ` +:file:`{code}.hist` the :ref:`history file ` +:file:`{code}.report` the :ref:`report file ` +:file:`{code}.cmd` the :ref:`remote control file ` +:file:`{code}.games/` |sgf| :ref:`game records ` +:file:`{code}.void/` |sgf| game records for :ref:`void games ` +:file:`{code}.gtplogs/` |gtp| logs + (from :option:`--log-gtp `) +======================= ======================================================= + +The recommended filename extension for the control file is :file:`.ctl`, but +other extensions are allowed (except those listed in the table above). + + +.. _competition state: + +Competition state +^^^^^^^^^^^^^^^^^ + +.. index:: state file + +The competition :dfn:`state file` (:file:`{code}.state`) contains a +machine-readable description of the competition's results; this allows +resuming the competition, and also programmatically :ref:`querying the results +`. It is rewritten after each game result is received, +so that little information will be lost if the ringmaster stops ungracefully +for any reason. + +The :action:`reset` command line action deletes **all** competition output +files, including game records and the state file. + +State files written by one Gomill release may not be accepted by other +releases. See :doc:`changes` for details. + +.. caution:: If the ringmaster loads a state file written by a hostile party, + it can be tricked into executing arbitrary code. On a shared system, do not + make the competition directory or the state file world-writeable. + + +.. index:: logging, event log, history file + +.. _logging: + +Logging +^^^^^^^ + +The ringmaster writes two log files: the :dfn:`event log` (:file:`{code}.log`) +and the :dfn:`history file` (:file:`{code}.hist`). + +The event log has entries for competition runs starting and finishing and for +games starting and finishing, including details of errors from games which +fail. It may also include output from the players' :ref:`standard error +streams `, depending on the :setting:`stderr_to_log` setting. + +The history file has entries for game results, and in tuning events it +may have periodic descriptions of the tuner status. + +Also, if the :option:`--log-gtp ` command line option is +passed, the ringmaster logs all |gtp| commands and responses. It writes a +separate log file for each game, in the :file:`{code}.gtplogs` directory. + + +.. _standard error: + +Players' standard error +^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the players' standard error streams are sent to the ringmaster's +:ref:`event log `. All players write to the same log, so there's no +direct indication of which messages came from which player (the log entries +for games starting and completing may help). + +If the competition setting :setting:`stderr_to_log` is False, the players' +standard error streams are left unchanged from the ringmaster's. This is only +useful in :ref:`quiet mode `, or if you redirect the ringmaster's +standard error. + +You can send standard error for a particular player to :file:`/dev/null` using +the player setting :setting:`discard_stderr`. This can be used for players +which like to send copious diagnostics to stderr, but if possible it is better +to configure the player not to do that, so that any real error messages aren't +hidden (eg with a command line option like ``fuego --quiet``). + diff --git a/gomill/docs/conf.py b/gomill/docs/conf.py new file mode 100644 index 0000000..537812d --- /dev/null +++ b/gomill/docs/conf.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +needs_sphinx = '1.0' +extensions = ['sphinx.ext.todo', 'sphinx.ext.pngmath', 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode'] +templates_path = ['_templates'] +source_suffix = '.rst' +source_encoding = 'utf-8' +master_doc = 'index' +project = u'gomill' +copyright = u'2009-2011, Matthew Woodcraft' +version = '0.7.2' +release = '0.7.2' +unused_docs = [] +exclude_dirnames = ['.git'] +pygments_style = 'vs' +modindex_common_prefix = ['gomill.'] + +html_theme = 'default' +html_theme_options = { + 'nosidebar' : False, + #'rightsidebar' : True, + 'stickysidebar' : False, + + 'footerbgcolor' : '#3d3011', + #'footertextcolor' : , + 'sidebarbgcolor' : '#3d3011', + #'sidebartextcolor' : , + 'sidebarlinkcolor' : '#d8d898', + 'relbarbgcolor' : '#523f13', + #'relbartextcolor' : , + #'relbarlinkcolor' : , + #'bgcolor' : , + #'textcolor' : , + 'linkcolor' : '#7c5f35', + 'visitedlinkcolor' : '#7c5f35', + #'headbgcolor' : , + 'headtextcolor' : '#5c4320', + #'headlinkcolor' : , + #'codebgcolor' : , + #'codetextcolor' : , + + 'externalrefs' : True, + } + +html_static_path = ['_static'] +html_add_permalinks = False +html_copy_source = False +html_sidebars = {'**' : ['wholetoc.html', 'relations.html', 'searchbox.html']} +html_style = "gomill.css" +html_show_sourcelink = False + +pngmath_use_preview = True + +todo_include_todos = True + +intersphinx_mapping = {'python': ('http://docs.python.org/2.7', + 'python-inv.txt')} + + +rst_epilog = """ +.. |gtp| replace:: :abbr:`GTP (Go Text Protocol)` +.. |sgf| replace:: :abbr:`SGF (Smart Game Format)` +""" + +def setup(app): + app.add_object_type('action', 'action', + indextemplate='pair: %s; ringmaster action', + objname="Ringmaster action") + + app.add_object_type('gtp', 'gtp', + indextemplate='pair: %s; GTP command', + objname="GTP command") + + app.add_object_type('script', 'script', + indextemplate='pair: %s; example script', + objname="Example script") + + app.add_object_type('setting', 'setting', + indextemplate='pair: %s; control file setting', + objname="Control file setting") + + app.add_object_type('pl-setting', 'pl-setting', + indextemplate='pair: %s; Playoff tournament setting', + objname="Playoff tournament setting") + + app.add_object_type('aa-setting', 'aa-setting', + indextemplate='pair: %s; All-play-all tournament setting', + objname="All-play-all tournament setting") + + app.add_object_type('mc-setting', 'mc-setting', + indextemplate='pair: %s; Monte Carlo tuner setting', + objname="Monte Carlo tuner setting") + + app.add_object_type('ce-setting', 'ce-setting', + indextemplate='pair: %s; cross-entropy tuner setting', + objname="Cross-entropy tuner setting") + + app.add_crossref_type('setting-cls', 'setting-cls', + indextemplate='single: %s', + objname="Control file object") + + app.add_crossref_type('pl-setting-cls', 'pl-setting-cls', + indextemplate='single: %s', + objname="Control file object") + + app.add_crossref_type('mc-setting-cls', 'mc-setting-cls', + indextemplate='single: %s', + objname="Control file object") + + app.add_crossref_type('ce-setting-cls', 'ce-setting-cls', + indextemplate='single: %s', + objname="Control file object") + + +# Undo undesirable sphinx code that auto-adds 'xref' class to literals 'True', +# 'False', and 'None'. +from sphinx.writers import html as html_mod +def visit_literal(self, node): + self.body.append(self.starttag(node, 'tt', '', + CLASS='docutils literal')) + self.protect_literal_text += 1 +html_mod.HTMLTranslator.visit_literal = visit_literal + diff --git a/gomill/docs/contact.rst b/gomill/docs/contact.rst new file mode 100644 index 0000000..3ef98b9 --- /dev/null +++ b/gomill/docs/contact.rst @@ -0,0 +1,13 @@ +Contact +======= + +Gomill's home page is http://mjw.woodcraft.me.uk/gomill/. + +Updated versions will be made available for download from that site. + +I'm happy to receive any bug reports, suggestions, patches, questions and so +on at . + +I'm particularly interested in hearing about any |gtp| engines (even buggy +ones) which don't work with the ringmaster. + diff --git a/gomill/docs/errors.rst b/gomill/docs/errors.rst new file mode 100644 index 0000000..59d5b26 --- /dev/null +++ b/gomill/docs/errors.rst @@ -0,0 +1,187 @@ +Error handling and exceptional situations +----------------------------------------- + +This page contains some tedious details of the implementation; it might be of +interest if you're wondering whether the behaviour you see is intentional or a +bug. + +.. contents:: Page contents + :local: + :backlinks: none + + +.. _game id: + +Game identification +^^^^^^^^^^^^^^^^^^^ + +Each game played in a competition is identified using a short string (the +:dfn:`game_id`). This is used in the |sgf| :ref:`game record ` +filename and game name (``GN``), the :ref:`log files `, the live +display, and so on. + +For playoff tournaments, game ids are made up from the :pl-setting:`matchup id +` and the number of the game within the matchup; for example, the first +game played might be ``0_0`` or ``0_000`` (depending on the value of +:pl-setting:`number_of_games`). + +Similarly for all-play-all tournaments, game ids are like ``AvB_0``, using the +competitor letters shown in the results grid, with the length depending on the +:aa-setting:`rounds` setting. + + +.. _details of scoring: + +Details of scoring +^^^^^^^^^^^^^^^^^^ + +If :setting:`scorer` is ``"players"`` but neither engine is able to score +(whether because :gtp:`!final_score` isn't implemented, or it fails, or +:setting:`is_reliable_scorer` is ``False``), the game result is reported as +unknown (|sgf| result ``?``). + +If both engines are able to score but they disagree about the winner, the game +result is reported as unknown. The engines' responses to :gtp:`!final_score` +are recorded in |sgf| file comments. + +If the engines agree about the winner but disagree about the winning margin, +the |sgf| result is simply ``B+`` or ``W+``, and the engines' responses are +recorded in |sgf| file comments. + + +.. _engine errors: + +Engine errors +^^^^^^^^^^^^^ + +If an engine returns a |gtp| failure response to any of the commands which set +up the game (eg :gtp:`!boardsize` or :gtp:`!fixed_handicap`), the game is +treated as :ref:`void `. + +If an engine fails to start, exits unexpectedly, or produces a |gtp| response +which is ill-formed at the protocol level, the game is treated as :ref:`void +`. + +As an exception, if such an error happens after the game's result has been +established (in particular, if one player has already forfeited the game), the +game is not treated as void. + + +.. _engine exit behaviour: + +Engine exit behaviour +^^^^^^^^^^^^^^^^^^^^^ + +Before reporting the game result, the ringmaster sends :gtp:`!quit` to both +engines, closes their input and output pipes, and waits for the subprocesses +to exit. + +If an engine hangs (during the game or at exit), the ringmaster will just hang +too (or, if in parallel mode, one worker process will). + +The exit status of engine subprocesses is ignored. + + +.. index:: void games + +.. _void games: + +Void games +^^^^^^^^^^ + +Void games are games which were not completed due to a software failure, and +which don't count as a forfeit by either engine. + +Void games don't appear in the competition results. They're recorded in the +:ref:`event log `, and a warning is displayed on screen when they +occur. + +If :setting:`record_games` is enabled, a game record will be written for each +void game that had at least one move played. These are placed in the +:file:`{code}.void/` subdirectory of the competition directory. + +A void game will normally be replayed, with the same game id (the details +depend on the competition type; see below). + +(Note that void games aren't the same thing as games whose |sgf| result is +``Void``; the ringmaster uses that result for games which exceed the +:setting:`move_limit`.) + + +Halting competitions due to errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A single error which causes a void game will not normally cause a competition +to be prematurely halted, but multiple errors may. + +The details depend on the competition type: + +For playoff and all-play-all tournaments, a run is halted early if the first +game in any matchup is void, or if two games in a row for the same matchup are +void. + +For tuning events, a run is halted immediately if the first game to finish is +void. + +Otherwise, in Monte Carlo tuning events a void game will be ignored: a new +game will be scheduled from the current state of the MCTS tree (and the +original game number will be skipped). If two game results in a row are void, +the run will be halted. + +In cross-entropy tuning events a void game will be replayed; if it fails +again, the run will be halted. + +In parallel mode, outstanding games will be allowed to complete. + + +Preventing simultaneous runs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If :c:func:`!flock()` is available, the ringmaster will detect attempts to run +a competition which is already running (but this probably won't work if the +control file is on a network filesystem). + +It's fine to use :action:`show` and :action:`report`, or the :doc:`tournament +results API `, while a competition is running. + + +Signals and controlling terminal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The check for :kbd:`Ctrl-X` uses the ringmaster's controlling terminal, +independently of stdin and stdout. If there's no controlling terminal, or +:mod:`termios` isn't available, this check is disabled. + +The engine subprocesses are left attached to the ringmaster's controlling +terminal, so they will receive signals from :kbd:`Ctrl-C`; unless they detach +from their controlling terminal or ignore the signal, they should exit +cleanly in response. + +Running the ringmaster in the background (including using :kbd:`Ctrl-Z`) +should work properly (you probably want :ref:`quiet mode `). + + +.. _remote control file: + +The remote control file +^^^^^^^^^^^^^^^^^^^^^^^ + +The :action:`stop` action is implemented by writing a :file:`{code}.cmd` file +to the competition directory. + + +Character encoding +^^^^^^^^^^^^^^^^^^ + +Gomill is designed for a UTF-8 environment; it is intended to work correctly +if non-ASCII characters provided as input are encoded in UTF-8, and to produce +terminal and report output in UTF-8. + +Non-ASCII characters in the control file must be encoded in UTF-8. + +|gtp| engines may return UTF-8 characters in in response to :gtp:`!name`, +:gtp:`!version`, :gtp:`gomill-describe_engine`, or +:gtp:`gomill-explain_last_move`. + +SGF files written by Gomill always explicitly specify UTF-8 encoding. + diff --git a/gomill/docs/example_scripts.rst b/gomill/docs/example_scripts.rst new file mode 100644 index 0000000..4ca22ba --- /dev/null +++ b/gomill/docs/example_scripts.rst @@ -0,0 +1,104 @@ +.. _example scripts: + +The example scripts +=================== + +The following example scripts are available in the :file:`gomill_examples/` +directory of the Gomill source distribution. + +Some of them may be independently useful, as well as illustrating the library +API. + +See the top of each script for further information. + +See :ref:`running the example scripts ` for notes +on making the :mod:`!gomill` package available for use with the example +scripts. + + +.. script:: show_sgf.py + + Prints an ASCII diagram of the position from an |sgf| file. + + This demonstrates the :mod:`~gomill.sgf`, :mod:`~gomill.sgf_moves`, and + :mod:`~gomill.ascii_boards` modules. + + +.. script:: split_sgf_collection.py + + Splits a file containing an |sgf| game collection into multiple files. + + This demonstrates the parsing functions from the :mod:`!sgf_grammar` module. + + +.. script:: twogtp + + A 'traditional' twogtp implementation. + + This demonstrates the :mod:`!gtp_games` module. + + +.. script:: find_forfeits.py + + Finds the forfeited games from a playoff or all-play-all tournament. + + This demonstrates the :doc:`tournament results API `. + + +.. script:: gtp_test_player + + A |gtp| engine intended for testing |gtp| controllers. + + This demonstrates the low-level engine-side |gtp| code (the + :mod:`!gtp_engine` module). + + +.. script:: gtp_stateful_player + + A |gtp| engine which maintains the board position. + + This demonstrates the :mod:`!gtp_states` module, which can be used to make a + |gtp| engine from a stateless move-generating program, or to add commands + like :gtp:`!undo` and :gtp:`!loadsgf` to an engine which doesn't natively + support them. + + +.. script:: kgs_proxy.py + + A |gtp| engine proxy intended for use with `kgsGtp`_. This produces game + records including the engine's commentary, if the engine supports + :gtp:`gomill-savesgf`. + + .. _`kgsGtp`: http://senseis.xmp.net/?KgsGtp + + This demonstrates the :mod:`!gtp_proxy` module, and may be independently + useful. + + +.. script:: mogo_wrapper.py + + A |gtp| engine proxy intended for use with `Mogo`_. This can be used to run + Mogo with a |gtp| controller (eg `Quarry`_) which doesn't get on with Mogo's + |gtp| implementation. + + .. _`Mogo`: http://www.lri.fr/~gelly/MoGo_Download.htm + .. _`Quarry`: http://home.gna.org/quarry/ + + This demonstrates the :mod:`!gtp_proxy` module, and may be independently + useful. + + +.. script:: gomill-clop + + An experimental script for using Gomill as a back end for Rémi Coulom's CLOP + optimisation system. It has been tested with ``CLOP-0.0.8``, which can be + downloaded from http://remi.coulom.free.fr/CLOP/ . + + To use it, write a control file based on :file:`clop_example.ctl` in the + :file:`gomill_examples/` directory, and run :: + + $ gomill-clop setup + + That will create a :samp:`.clop` file in the same directory as the control + file, which you can then run using :samp:`clop-gui`. + diff --git a/gomill/docs/glossary.rst b/gomill/docs/glossary.rst new file mode 100644 index 0000000..fb18e9e --- /dev/null +++ b/gomill/docs/glossary.rst @@ -0,0 +1,116 @@ +Glossary +======== + +.. glossary:: + + GTP + The Go Text Protocol + + A communication protocol used to control Go-playing programs. Gomill + uses only GTP version 2, which is specified at + http://www.lysator.liu.se/~gunnar/gtp/gtp2-spec-draft2/gtp2-spec.html. + + (As of August 2011, the specification describes itself as a draft, but it + has remained stable for several years and is widely implemented.) + + + SGF + The Smart Game Format + + A text-based file format used for storing Go game records. + + Gomill uses version FF[4], which is specified at + http://www.red-bean.com/sgf/index.html. + + + jigo + A tied game (after komi is taken into account). + + + komi + Additional points awarded to White in final scoring. + + + simple ko + A Go rule prohibiting repetition of the immediately-preceding position. + + + superko + A Go rule prohibiting repetition of preceding positions. + + There are several possible variants of the superko rule. Gomill does not + enforce any of them. + + + pondering + A feature implemented by some Go programs: thinking while it is their + opponent's turn to move. + + + controller + A program implementing the 'referee' side of the |gtp| protocol. + + The |gtp| protocol can be seen as a client-server protocol, with the + controller as the client. + + + engine + A program implementing the 'playing' side of the |gtp| protocol. + + The |gtp| protocol can be seen as a client-server protocol, with the + engine as the server. + + + player + A |gtp| engine, together with a particular configuration. + + + competition + An 'event' consisting of multiple games managed by the Gomill ringmaster + (either a tournament or a tuning event). + + tournament + A competition in which the ringmaster plays games between predefined + players, to compare their strengths. + + playoff + A tournament comprising many games played between fixed pairings of + players. + + all-play-all + A tournament in which games are played between all pairings from a list of + players. + + + matchup + A pairing of players in a tournament, together with its settings (board + size, komi, handicap, and so on) + + + tuning event + A competition in which the ringmaster runs an algorithm which adjusts + player parameters to try to find the values which give strongest play. + + + Bandit problem + A problem in which an agent has to repeatedly choose between actions whose + value is initially unknown, trading off time spent on the action with the + best estimated value against time spent evaluating other actions. + + See http://en.wikipedia.org/wiki/Multi-armed_bandit + + + UCB + Upper Confidence Bound algorithms + + A family of algorithms for addressing bandit problems. + + + UCT + Upper Confidence bounds applied to Trees. + + A variant of UCB for bandit problems in which the actions are arranged in + the form of a tree. + + See http://senseis.xmp.net/?UCT. + diff --git a/gomill/docs/gomill_package.rst b/gomill/docs/gomill_package.rst new file mode 100644 index 0000000..41ecb78 --- /dev/null +++ b/gomill/docs/gomill_package.rst @@ -0,0 +1,79 @@ +The :mod:`gomill` package +------------------------- + +All Gomill code is contained in modules under the :mod:`!gomill` package. + +The package includes both the 'toolkit' (Go board, |sgf|, and |gtp|) code, and +the code implementing the ringmaster. + +.. contents:: Page contents + :local: + :backlinks: none + + +Package module contents +^^^^^^^^^^^^^^^^^^^^^^^ + +The package module itself defines only a single constant: + +.. module:: gomill + :synopsis: Tools for testing and tuning Go-playing programs. + +.. data:: __version__ + + The library version, as a string (like ``"0.7"``). + + .. versionadded:: 0.7 + + +Generic data representation +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Unless otherwise stated, string values are 8-bit UTF-8 strings. + + +.. _go_related_data_representation: + +Go-related data representation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Gomill represents Go colours and moves internally as follows: + +======== =========================================== + Name Possible values +======== =========================================== +*colour* single-character string: ``'b'`` or ``'w'`` +*point* pair (*int*, *int*) of coordinates +*move* *point* or ``None`` (for a pass) +======== =========================================== + +The terms *colour*, *point*, and *move* are used as above throughout this +library documentation (in particular, when describing parameters and return +types). + +*colour* values are used to represent players, as well as stones on the board. +(When a way to represent an empty point is needed, ``None`` is used.) + +*point* values are treated as (row, column). The bottom left is ``(0, 0)`` +(the same orientation as |gtp|, but not |sgf|). So the coordinates for a 9x9 +board are as follows:: + + 9 (8,0) . . . . . (8,8) + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 . . . . . . . . . + 2 . . . . . . . . . + 1 (0,0) . . . . . (0,8) + A B C D E F G H J + +There are functions in the :mod:`~gomill.common` module to convert between +these coordinates and the conventional (``T19``\ -style) notation. + +Gomill is designed to work with square boards, up to 25x25 (which is the upper +limit of the conventional notation, and the upper limit for |gtp|). Some parts +of the library can work with larger board sizes; these cases are documented +explicitly. + diff --git a/gomill/docs/gtp_extensions.rst b/gomill/docs/gtp_extensions.rst new file mode 100644 index 0000000..bf9dfad --- /dev/null +++ b/gomill/docs/gtp_extensions.rst @@ -0,0 +1,133 @@ +GTP extensions +============== + +Gomill supports a number of |gtp| extension commands. These are all named with +a ``gomill-`` prefix. + + +The extensions used by the ringmaster are as follows: + +.. gtp:: gomill-explain_last_move + + Arguments: none + + Return a string containing the engine's comments about the last move it + generated. + + This might include the engine's estimate of its winning chances, a principal + variation, or any other diagnostic information. + + The intention is that |gtp| controllers which produce game records should + use this command to write a comment associated with the move. + + Any non-ASCII characters in the response should be encoded as UTF-8. + + If no information is available, return an empty string. + + The behaviour of this command is unspecified if a command changing the board + state (eg :gtp:`!play` or :gtp:`!undo`) has occurred since the engine last + generated a move. + + +.. gtp:: gomill-describe_engine + + Arguments: none + + Return a string with a description of the engine's configuration. This + should repeat the information from the :gtp:`!name` and :gtp:`!version` + commands. Controllers should expect the response to take multiple lines. + + The intention is that |gtp| controllers which produce game records should + use the output of this command as part of a comment for the game as a whole. + + If possible, the response should include a description of all engine + parameters which affect gameplay. If the engine plays reproducibly given the + seed of a random number generator, the response should include that seed. + + Any non-ASCII characters in the response should be encoded as UTF-8. + + +.. gtp:: gomill-cpu_time + + Arguments: none + + Return a float (represented in decimal) giving the amount of CPU time the + engine has used to generate all moves made so far (in seconds). + + For engines which use multiple threads or processes, this should be the + total time used on all CPUs. + + It may not be possible to meaningfully respond to this command (for example, + if an engine runs on multiple processors which run at different speeds); in + complex cases, the engine should document how the CPU time is calculated. + + +.. gtp:: gomill-genmove_ex + + Arguments: colour, list of keywords + + This is a variant of the standard :gtp:`!genmove` command. Each keyword + indicates a permitted (or desired) variation of behaviour. For example:: + + gomill-genmove_ex b claim + + If :gtp:`!gomill-genmove_ex` is sent without any arguments (ie, no colour is + specified), the engine should return a list of the keywords it supports (one + per line, like :gtp:`!list_commands`). + + Engines must ignore keywords they do not support. :gtp:`!gomill-genmove_ex` + with no keywords is exactly equivalent to :gtp:`!genmove`. + + The following keywords are currently defined: + + ``claim`` + In addition to the usual responses to :gtp:`!genmove`, the engine may also + return ``claim``, which indicates that the engine believes it is certain + to win the game (the engine must not assume that the controller will act + on this claim). + + +There is also an extension which is not used by the ringmaster: + +.. gtp:: gomill-savesgf + + Arguments: filename, list of |sgf| properties + + Write an |sgf| game record of the current game. + + See the :term:`GTP` specification's description of :gtp:`!loadsgf` for the + interpretation of the ``filename`` argument. + + The |sgf| properties should be specified in the form + :samp:`{PropIdent}={PropValue}`, eg ``RE=W+3.5``. Escape spaces in values + with ``\_``, backslashes with ``\\``. Encode non-ASCII characters in UTF-8. + + These |sgf| properties should be added to the root node. The engine should + fill in any properties it can (at least ``AP``, ``SZ``, ``KM``, ``HA``, and + ``DT``). Explicitly-specified properties should override the engine's + defaults. + + The intention is that engines which have 'comments' about their moves (as + for :gtp:`gomill-explain_last_move`) should include them in the game record. + + Example:: + + gomill-savesgf xxx.sgf PB=testplayer PW=GNU\_Go:3.8 RE=W+3.5 + + .. note:: + + |gtp| engines aren't typically well placed to write game records, as they + don't have enough information to write the game metadata properly (this is + why :gtp:`!gomill-savesgf` has to take the |sgf| properties explicitly). + It's usually better for the controller to do it. See the + :script:`kgs_proxy.py` example script for an example of when this command + might be useful. + + +The :gtp:`gomill-explain_last_move`, :gtp:`gomill-genmove_ex`, and +:gtp:`gomill-savesgf` commands are supported by the Gomill :mod:`!gtp_states` +module. + +.. The other extension is gomill-passthrough (used by proxies), but I don't + think it makes sense to document it as a generic extension + diff --git a/gomill/docs/handicap_layout.rst b/gomill/docs/handicap_layout.rst new file mode 100644 index 0000000..fe8ce22 --- /dev/null +++ b/gomill/docs/handicap_layout.rst @@ -0,0 +1,36 @@ +The :mod:`~gomill.handicap_layout` module +----------------------------------------- + +.. module:: gomill.handicap_layout + :synopsis: Standard layout of fixed handicap stones. + +The :mod:`!gomill.handicap_layout` module describes the standard layout used +for fixed handicap stones. It follows the rules from the :term:`GTP` +specification. + + +.. function:: handicap_points(number_of_stones, board_size) + + :rtype: list of *points* + + Returns the handicap points for a given number of stones and board size. + + Raises :exc:`ValueError` if there isn't a standard placement pattern for + the specified number of handicap stones and board size. + + The result's length is always exactly *number_of_stones*. + +.. function:: max_fixed_handicap_for_board_size(board_size) + + :rtype: int + + Returns the maximum number of stones permitted for the |gtp| + :gtp:`!fixed_handicap` command, given the specified board size. + +.. function:: max_free_handicap_for_board_size(board_size) + + :rtype: int + + Returns the maximum number of stones permitted for the |gtp| + :gtp:`!place_free_handicap` command, given the specified board size. + diff --git a/gomill/docs/index.rst b/gomill/docs/index.rst new file mode 100644 index 0000000..5509df6 --- /dev/null +++ b/gomill/docs/index.rst @@ -0,0 +1,24 @@ +****** +Gomill +****** + +.. toctree:: + :maxdepth: 3 + :titlesonly: + + intro + ringmaster + gtp_extensions + library + example_scripts + install + contact + changes + licence + glossary + +* :ref:`genindex` +* :ref:`search` + + +.. todolist:: diff --git a/gomill/docs/install.rst b/gomill/docs/install.rst new file mode 100644 index 0000000..0a332e1 --- /dev/null +++ b/gomill/docs/install.rst @@ -0,0 +1,152 @@ +Installation +============ + +.. contents:: Page contents + :local: + :backlinks: none + + +Requirements +------------ + +Gomill requires Python 2.5, 2.6, or 2.7. + +For Python 2.5 only, the :option:`--parallel ` feature +requires the external `multiprocessing`__ package. + +.. __: http://pypi.python.org/pypi/multiprocessing + +Gomill is intended to run on any modern Unix-like system. + + +Obtaining Gomill +---------------- + +Gomill is distributed as a pure-Python source archive, +:file:`gomill-{version}.tar.gz`. The most recent version can be obtained from +http://mjw.woodcraft.me.uk/gomill/. + +This documentation is distributed separately as +:file:`gomill-doc-{version}.tar.gz`. + +Once you have downloaded the source archive, extract it using a command like +:samp:`tar -xzf gomill-{version}.tar.gz`. This will create a directory named +:file:`gomill-{version}`, referred to below as the :dfn:`distribution +directory`. + +Alternatively, you can access releases using Git:: + + git clone http://mjw.woodcraft.me.uk/gomill/git/ gomill + +which would create :file:`gomill` as the distribution directory. + + + +Running the ringmaster +---------------------- + +The ringmaster executable in the distribution directory can be run directly +without any further installation; it will use the copy of the :mod:`!gomill` +package in the distribution directory. + +A symbolic link to the ringmaster executable will also work, but if you move +the executable elsewhere it will not be able to find the :mod:`!gomill` +package unless the package is installed. + + +Installing +---------- + +Installing Gomill puts the :mod:`!gomill` package onto the Python module +search path, and the ringmaster executable onto the executable +:envvar:`!PATH`. + +To install, first change to the distribution directory, then: + +- to install for the system as a whole, run (as a sufficiently privileged + user) :: + + python setup.py install + + +- to install for the current user only (Python 2.6 or 2.7), run :: + + python setup.py install --user + + (in this case the ringmaster executable will be placed in + :file:`~/.local/bin`.) + +Pass :option:`!--dry-run` to see what these will do. See +http://docs.python.org/2.7/install/ for more information. + + +Uninstalling +------------ + +To remove an installed version of Gomill, run :: + + python setup.py uninstall + +(This uses the Python module search path and the executable :envvar:`!PATH` to +find the files to remove; pass :option:`!--dry-run` to see what it will do.) + + + +Running the test suite +---------------------- + +To run the testsuite against the distributed :mod:`!gomill` package, change to +the distribution directory and run :: + + python -m gomill_tests.run_gomill_testsuite + + +To run the testsuite against an installed :mod:`!gomill` package, change to +the distribution directory and run :: + + python test_installed_gomill.py + + +With Python versions earlier than 2.7, the unittest2__ library is required +to run the testsuite. + +.. __: http://pypi.python.org/pypi/unittest2/ + + +.. _running the example scripts: + +Running the example scripts +--------------------------- + +To run the example scripts, it is simplest to install the :mod:`!gomill` +package first. + +If you do not wish to do so, you can run :: + + export PYTHONPATH= + +so that the example scripts will be able to find the :mod:`!gomill` package. + + + +Building the documentation +-------------------------- + +The sources for this HTML documentation are included in the Gomill source +archive. To rebuild the documentation, change to the distribution directory +and run :: + + python setup.py build_sphinx + +The documentation will be generated in :file:`build/sphinx/html`. + +Requirements: + +- Sphinx__ version 1.0 or later (at least 1.0.4 recommended) +- LaTeX__ +- dvipng__ + +.. __: http://sphinx.pocoo.org/ +.. __: http://www.latex-project.org/ +.. __: http://www.nongnu.org/dvipng/ + diff --git a/gomill/docs/intro.rst b/gomill/docs/intro.rst new file mode 100644 index 0000000..b5bc761 --- /dev/null +++ b/gomill/docs/intro.rst @@ -0,0 +1,97 @@ +Introduction +============ + +Gomill is a suite of tools, and a Python library, for use in developing and +testing Go-playing programs. It is based around the Go Text Protocol +(:term:`GTP`) and the Smart Game Format (:term:`SGF`). + +The principal tool is the :dfn:`ringmaster`, which plays programs against each +other and keeps track of the results. + +Ringmaster features include: + +- testing multiple pairings in one run +- playing multiple games in parallel +- displaying live results +- engine configuration by command line options or |gtp| commands +- a protocol for including per-move engine diagnostics in |sgf| output +- automatically tuning engine parameters based on game results + (**experimental**) + +.. contents:: Page contents + :local: + :backlinks: none + + +Ringmaster example +------------------ + +Create a file called :file:`demo.ctl`, with the following contents:: + + competition_type = 'playoff' + + board_size = 9 + komi = 7.5 + + players = { + 'gnugo-l1' : Player('gnugo --mode=gtp --level=1'), + 'gnugo-l2' : Player('gnugo --mode=gtp --level=2'), + } + + matchups = [ + Matchup('gnugo-l1', 'gnugo-l2', + alternating=True, + number_of_games=5), + ] + +(If you don't have :program:`gnugo` installed, change the +:setting-cls:`Player` definitions to use a command line for whatever |gtp| +engine you have available.) + +Then run :: + + $ ringmaster demo.ctl + +The ringmaster will run five games between the two players, showing a summary +of the results on screen, and then exit. + +(If the ringmaster is not already installed, see :doc:`install` for +instructions.) + +The final display should be something like this:: + + gnugo-l1 v gnugo-l2 (5/5 games) + board size: 9 komi: 7.5 + wins black white avg cpu + gnugo-l1 2 40.00% 1 33.33% 1 50.00% 1.05 + gnugo-l2 3 60.00% 1 50.00% 2 66.67% 1.12 + 2 40.00% 3 60.00% + + = Results = + game 0_0: gnugo-l2 beat gnugo-l1 W+21.5 + game 0_1: gnugo-l2 beat gnugo-l1 B+9.5 + game 0_2: gnugo-l2 beat gnugo-l1 W+14.5 + game 0_3: gnugo-l1 beat gnugo-l2 W+7.5 + game 0_4: gnugo-l1 beat gnugo-l2 B+2.5 + +The ringmaster will create several files named like :file:`demo.{xxx}` in the +same directory as :file:`demo.ctl`, including a :file:`demo.sgf` directory +containing game records. + + +The Python library +------------------ + +Gomill also provides a Python library for working with |gtp| and |sgf|, though +as of Gomill |version| only part of the API is stable. See :doc:`library` for +details. + + +The example scripts +------------------- + +Some :doc:`example scripts ` are also included in the Gomill +distribution, as illustrations of the library interface and in some cases as +tools useful in themselves. + + diff --git a/gomill/docs/library.rst b/gomill/docs/library.rst new file mode 100644 index 0000000..85b53c9 --- /dev/null +++ b/gomill/docs/library.rst @@ -0,0 +1,32 @@ +The Gomill library +================== + +Gomill is intended to be useful as a Python library for developing |gtp|- and +|sgf|-based tools. + +As of Gomill |version|, not all of the library API is formally documented. +Only the parts which are described in this documentation should be considered +stable public interfaces. + +Nonetheless, the source files for the remaining modules contain fairly +detailed documentation, and the :doc:`example scripts ` +illustrate how some of them can be used. + +.. contents:: Page contents + :local: + :backlinks: none + + +.. toctree:: + :maxdepth: 3 + :titlesonly: + + library_overview + gomill_package + common + boards + ascii_boards + handicap_layout + sgf + tournament_results + diff --git a/gomill/docs/library_overview.rst b/gomill/docs/library_overview.rst new file mode 100644 index 0000000..f880d1d --- /dev/null +++ b/gomill/docs/library_overview.rst @@ -0,0 +1,74 @@ +Library overview +---------------- + +The :mod:`gomill` package includes the following modules: + +.. the descriptions here should normally match the module :synopsis:, and + therefore the module index. + +========================================= ======================================================================== +Generic support code +========================================= ======================================================================== +:mod:`~!gomill.utils` +:mod:`~!gomill.compact_tracebacks` +:mod:`~!gomill.ascii_tables` +:mod:`~!gomill.job_manager` +:mod:`~!gomill.settings` +========================================= ======================================================================== + +========================================= ======================================================================== + Go-related support code +========================================= ======================================================================== +:mod:`~gomill.common` Go-related utility functions. +:mod:`~gomill.boards` Go board representation. +:mod:`~gomill.ascii_boards` ASCII Go board diagrams. +:mod:`~gomill.handicap_layout` Standard layout of fixed handicap stones. +========================================= ======================================================================== + +========================================= ======================================================================== + |sgf| support +========================================= ======================================================================== +:mod:`~!gomill.sgf_grammar` +:mod:`~!gomill.sgf_properties` +:mod:`~gomill.sgf` High level |sgf| interface. +:mod:`~gomill.sgf_moves` Higher-level processing of moves and positions from |sgf| games +========================================= ======================================================================== + +========================================= ======================================================================== + |gtp| controller side +========================================= ======================================================================== +:mod:`~!gomill.gtp_controller` +:mod:`~!gomill.gtp_games` +========================================= ======================================================================== + +========================================= ======================================================================== + |gtp| engine side +========================================= ======================================================================== +:mod:`~!gomill.gtp_engine` +:mod:`~!gomill.gtp_states` +:mod:`~!gomill.gtp_proxy` +========================================= ======================================================================== + +========================================= ======================================================================== + Competitions +========================================= ======================================================================== +:mod:`~!gomill.competition_schedulers` +:mod:`~!gomill.competitions` +:mod:`~gomill.tournament_results` Retrieving and reporting on tournament results. +:mod:`~!gomill.tournaments` +:mod:`~!gomill.playoffs` +:mod:`~!gomill.allplayalls` +:mod:`~!gomill.cem_tuners` +:mod:`~!gomill.mcts_tuners` +========================================= ======================================================================== + +========================================= ======================================================================== + The ringmaster +========================================= ======================================================================== +:mod:`~!gomill.game_jobs` +:mod:`~!gomill.terminal_input` +:mod:`~!gomill.ringmaster_presenters` +:mod:`~!gomill.ringmasters` +:mod:`~!gomill.ringmaster_command_line` +========================================= ======================================================================== + diff --git a/gomill/docs/licence.rst b/gomill/docs/licence.rst new file mode 100644 index 0000000..b202821 --- /dev/null +++ b/gomill/docs/licence.rst @@ -0,0 +1,25 @@ +Licence +======= + +Gomill is copyright 2009-2011 Matthew Woodcraft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +.. Note:: This is the licence commonly known as the 'MIT' Licence. + diff --git a/gomill/docs/mcts_tuner.rst b/gomill/docs/mcts_tuner.rst new file mode 100644 index 0000000..09c7c07 --- /dev/null +++ b/gomill/docs/mcts_tuner.rst @@ -0,0 +1,646 @@ +.. index:: monte carlo tuner + +The Monte Carlo tuner +^^^^^^^^^^^^^^^^^^^^^ + +:setting:`competition_type` string: ``"mc_tuner"``. + +The Monte Carlo tuner treats the tuning event as a :term:`bandit problem`. +That is, it attempts to find the candidate which has the highest probability +of beating the opponent, and arranges to 'spend' more games on the candidates +which have the highest winning percentages so far. + +It does this using a form of the :term:`UCB` algorithm (or, optionally, +:term:`UCT`) which is familiar to Go programmers. + +.. caution:: As of Gomill |version|, the Monte Carlo tuner is still + experimental. The control file settings may change in future. The reports + aren't very good. + +.. contents:: Page contents + :local: + :backlinks: none + + +.. _mc parameter model: + +The parameter model +""""""""""""""""""" + +The Monte Carlo tuner expects to work with one or more independent player +parameters. + +Internally, it models each parameter value as a floating point number in the +range 0.0 to 1.0. It uses parameter values taken uniformly from this range to +make the candidate players. Values from this range are known as +:dfn:`optimiser parameters`. + +In practice, engine parameters might not be floating point numbers, their +range is unlikely to be 0.0 to 1.0, and you may wish to use a non-uniform (eg, +logarithmic) scale for the candidates. + +To support this, each parameter has an associated :mc-setting:`scale`. This is +a function which maps an optimiser parameter to an :dfn:`engine parameter` +(which can be of an arbitrary Python type). A number of :ref:`predefined +scales ` are provided. + +The candidate players are configured using these engine parameters. + +Reports, and the live display, are also based on engine parameters; see the +:mc-setting:`format` parameter setting. + + +Candidates +"""""""""" + +Each parameter also has a :mc-setting:`split` setting (a smallish integer). +This determines how many 'samples' of the parameter range are used to make +candidate players. + +When there are multiple parameters, one candidate is made for each combination +of these samples. So if there is only one parameter, the total number of +candidates is just :mc-setting:`split`, and if there are multiple parameters, +the total number of candidates is the product of all the :mc-setting:`split` +settings. For example, the sample control file below creates 64 candidates. + +.. caution:: While the Monte Carlo tuner does not impose any limit on the + number of parameters you use, unless the games are unusually rapid it may + be unreasonable to try to tune more than two or three parameters at once. + +Each candidate's engine parameters are passed to the +:mc-setting:`make_candidate` function, which returns a :setting-cls:`Player` +definition. + +The samples are taken by dividing the optimiser parameter range into +:mc-setting:`split` divisions, and taking the centre of each division as the +sample (so the end points of the range are not used). For example, if a +parameter has a linear scale from 0.0 to 8.0, and :mc-setting:`split` is 3, +the samples (after translation to engine parameters) will be 1.0, 4.0, and +7.0. + + +.. _the mcts tuning algorithm: + +The tuning algorithm +"""""""""""""""""""" + +Each time the tuner starts a new game, it chooses the candidate which gives +the highest value to the following formula: + +.. math:: w_c/g_c + E \sqrt(log(g_p) / g_c) + +where + +- :math:`E` is the :mc-setting:`exploration_coefficient` + +- :math:`g_c` is the number of games the candidate has played + +- :math:`w_c` is the number of games the candidate has won + +- :math:`g_p` is the total number of games played in the tuning event + +At the start of the tuning event, each candidate's :math:`g_c` is set to +:mc-setting:`initial_visits`, and :math:`w_c` is set to +:mc-setting:`initial_wins`. + +(:math:`w_c/g_c` is just the candidate's current win rate. :math:`E +\sqrt(log(g_p) / g_c)` is known as the :dfn:`exploration term`; as more games +are played, its value increases most rapidly for the least used candidates, so +that unpromising candidates will eventually be reconsidered.) + +When more than one candidate has the highest value (for example, at the start +of the event), one is chosen at random. + + +The tuning event runs until :mc-setting:`number_of_games` games have been +played (indefinitely, if :mc-setting:`number_of_games` is unset). + +The tuner can be stopped at any time; after each game result, it reports the +parameters of the current 'best' candidate. This is the candidate with the +most *wins* (note that this may not be the one with the best win rate; it is +usually the same as the candidate which has played the most games). + + + +.. _sample_mcts_control_file: + +Sample control file +""""""""""""""""""" + +Here is a sample control file, illustrating most of the available settings for +a Monte Carlo tuning event:: + + competition_type = "mc_tuner" + + description = """\ + This is a sample control file. + + It illustrates the available settings for the Monte Carlo tuner. + """ + + players = { + 'gnugo-l10' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=10"), + } + + def fuego(max_games, additional_commands=[]): + commands = [ + "go_param timelimit 999999", + "uct_max_memory 350000000", + "uct_param_search number_threads 1", + "uct_param_player reuse_subtree 0", + "uct_param_player ponder 0", + "uct_param_player max_games %d" % max_games, + ] + return Player( + "fuego --quiet", + startup_gtp_commands=commands+additional_commands) + + FUEGO_MAX_GAMES = 5000 + + parameters = [ + Parameter('rave_weight_initial', + scale = LOG(0.01, 5.0), + split = 8, + format = "I: %4.2f"), + + Parameter('rave_weight_final', + scale = LOG(1e2, 1e5), + split = 8, + format = "F: %4.2f"), + ] + + def make_candidate(rwi, rwf): + return fuego( + FUEGO_MAX_GAMES, + ["uct_param_search rave_weight_initial %f" % rwi, + "uct_param_search rave_weight_final %f" % rwf]) + + board_size = 19 + komi = 7.5 + opponent = 'gnugo-l10' + candidate_colour = 'w' + number_of_games = 10000 + + exploration_coefficient = 0.45 + initial_visits = 10 + initial_wins = 5 + + summary_spec = [40] + log_tree_to_history_period = 200 + + +.. _mcts_control_file_settings: + +Control file settings +""""""""""""""""""""" + +The following settings can be set at the top level of the control file: + +All :ref:`common settings ` (the :setting:`players` +dictionary is required, though it is used only to define the opponent). + +The following game settings (only :setting:`!board_size` and :setting:`!komi` +are required): + +- :setting:`board_size` +- :setting:`komi` +- :setting:`handicap` +- :setting:`handicap_style` +- :setting:`move_limit` +- :setting:`scorer` + +:setting:`!komi` must be fractional, as the tuning algorithm doesn't currently +support :term:`jigos `. + + +The following additional settings (all those without a listed default are +required): + +.. mc-setting:: number_of_games + + Integer (default ``None``) + + The total number of games to play in the event. If you leave this unset, + there will be no limit. + + +.. mc-setting:: candidate_colour + + String: ``"b"`` or ``"w"`` + + The colour for the candidates to take in every game. + + +.. mc-setting:: opponent + + Identifier + + The :ref:`player code ` of the player to use as the + candidates' opponent. + + +.. mc-setting:: parameters + + List of :mc-setting-cls:`Parameter` definitions (see :ref:`mc parameter + configuration`). + + Describes the parameter space that the tuner will work in. See :ref:`The + parameter model ` for more details. + + The order of the :mc-setting-cls:`Parameter` definitions is used for the + arguments to :mc-setting:`make_candidate`, and whenever parameters are + described in reports or game records. + + +.. mc-setting:: make_candidate + + Python function + + Function to create a :setting-cls:`Player` from its engine parameters. + + This function is passed one argument for each candidate parameter, and must + return a :setting-cls:`Player` definition. Each argument is the output of + the corresponding :mc-setting-cls:`Parameter`'s :mc-setting:`scale`. + + The function will typically use its arguments to construct command line + options or |gtp| commands for the player. For example:: + + def make_candidate(param1, param2): + return Player(["goplayer", "--param1", str(param1), + "--param2", str(param2)]) + + def make_candidate(param1, param2): + return Player("goplayer", startup_gtp_commands=[ + ["param1", str(param1)], + ["param2", str(param2)], + ]) + + +.. mc-setting:: exploration_coefficient + + Float + + The coefficient of the exploration term in the :term:`UCB` algorithm (eg + ``0.45``). See :ref:`The tuning algorithm `. + + +.. mc-setting:: initial_visits + + Positive integer + + The number of games to initialise each candidate with. At the start of the + event, the tuner will behave as if each candidate has already played this + many games. See :ref:`The tuning algorithm `. + + +.. mc-setting:: initial_wins + + Positive integer + + The number of wins to initialise each candidate with. At the start of the + event, the tuner will behave as if each candidate has already won this many + games. See :ref:`The tuning algorithm `. + + .. tip:: It's best to set :mc-setting:`initial_wins` so that + :mc-setting:`initial_wins` / :mc-setting:`initial_visits` is close to the + typical candidate's expected win rate. + + +.. mc-setting:: max_depth + + Positive integer (default 1) + + See :ref:`tree search` below. + + +The remaining settings only affect reporting and logging; they have no effect +on the tuning algorithm. + +.. mc-setting:: summary_spec + + List of integers (default [30]) + + Number of candidates to describe in the runtime display and reports (the + candidates with most visits are described). + + (This list should have :mc-setting:`max_depth` elements; if + :mc-setting:`max_depth` is greater than 1, it specifies how many candidates + to show from each level of the tree, starting with the highest.) + + +.. mc-setting:: log_tree_to_history_period + + Positive integer (default None) + + If this is set, a detailed description of the :term:`UCT` tree is written to + the :ref:`history file ` periodically (after every + :mc-setting:`!log_tree_to_history_period` games). + + +.. mc-setting:: number_of_running_simulations_to_show + + Positive integer (default 12) + + The maximum number of games in progress to describe on the runtime display. + + +.. _mc parameter configuration: + +Parameter configuration +""""""""""""""""""""""" + +.. mc-setting-cls:: Parameter + +A :mc-setting-cls:`!Parameter` definition has the same syntax as a Python +function call: :samp:`Parameter({arguments})`. Apart from :mc-setting:`!code`, +the arguments should be specified using keyword form (see +:ref:`sample_mcts_control_file`). + +All arguments other than :mc-setting:`format` are required. + +The arguments are: + + +.. mc-setting:: code + + Identifier + + A short string used to identify the parameter. This is used in error + messages, and in the default for :mc-setting:`format`. + + +.. mc-setting:: scale + + Python function + + Function mapping an optimiser parameter to an engine parameter; see :ref:`mc + parameter model`. + + Although this can be defined explicitly, in most cases you should be able + to use one of the :ref:`predefined scales `. + + Examples:: + + Parameter('p1', split = 8, + scale = LINEAR(-1.0, 1.0)) + + Parameter('p2', split = 8, + scale = LOG(10, 10000, integer=True)) + + Parameter('p3', split = 3, + scale = EXPLICIT(['low', 'medium', 'high'])) + + def scale_p3(f): + return int(1000 * math.sqrt(f)) + Parameter('p3', split = 20, scale = scale_p3) + + + +.. mc-setting:: split + + Positive integer + + The number of samples from this parameter to use to make candidates. See + :ref:`The tuning algorithm `. + + +.. mc-setting:: format + + String (default :samp:`"{parameter_code}: %s"`) + + Format string used to display the parameter value. This should include a + short abbreviation to indicate which parameter is being displayed, and also + contain ``%s``, which will be replaced with the engine parameter value. + + You can use any Python conversion specifier instead of ``%s``. For example, + ``%.2f`` will format a floating point number to two decimal places. ``%s`` + should be safe to use for all types of value. See `string formatting + operations`__ for details. + + .. __: http://docs.python.org/release/2.7/library/stdtypes.html#string-formatting-operations + + Format strings should be kept short, as screen space is limited. + + Examples:: + + Parameter('parameter_1', split = 8, + scale = LINEAR(-1.0, 1.0), + format = "p1: %.2f") + + Parameter('parameter_2', split = 8, + scale = LOG(10, 10000, integer=True), + format = "p2: %d") + + Parameter('parameter_3', split = 3, + scale = EXPLICIT(['low', 'medium', 'high']), + format = "p3: %s") + + +.. index:: predefined scale +.. index:: scale; predefined + +.. _predefined scales: + +Predefined scales +""""""""""""""""" + +There are three kinds of predefined scale which you can use in a +:mc-setting:`scale` definition: + +.. index:: LINEAR + +.. object:: LINEAR + + A linear scale between specified bounds. This takes two arguments: + ``lower_bound`` and ``upper_bound``. + + Optionally, you can also pass ``integer=True``, in which case the result is + rounded to the nearest integer. + + Examples:: + + LINEAR(0, 100) + LINEAR(-64.0, 256.0, integer=True) + + .. tip:: To make candidates which take each value from a simple integer range + from (say) 0 to 10 inclusive, use:: + + Parameter('p1', split = 11, + scale = LINEAR(-0.5, 10.5, integer=True)) + + (or use EXPLICIT) + + +.. index:: LOG + +.. object:: LOG + + A 'logarithmic scale' (ie, an exponential function) between specified + bounds. This takes two arguments: ``lower_bound`` and ``upper_bound``. + + Optionally, you can also pass ``integer=True``, in which case the result is + rounded to the nearest integer. + + Example:: + + LOG(0.01, 1000) + LOG(1e2, 1e9, integer=True) + + +.. index:: EXPLICIT + +.. object:: EXPLICIT + + This scale makes the engine parameters take values from an explicitly + specified list. You should normally use this with :mc-setting:`split` equal + to the length of the list. + + Examples:: + + EXPLICIT([0, 1, 2, 4, 6, 8, 10, 15, 20]) + EXPLICIT(['low', 'medium', 'high']) + + + .. note:: if :mc-setting:`max_depth` is greater than 1, :mc-setting:`split` + ^ :mc-setting:`max_depth` should equal the length of the list. + + +Writing scale functions +""""""""""""""""""""""" + +The following built-in Python functions might be useful: :func:`abs`, +:func:`min`, :func:`max`, :func:`round`. + +More functions are available from the :mod:`math` module. Put a line like :: + + from math import log, exp, sqrt + +in the control file to use them. + +Dividing two integers with ``/`` gives a floating point number (that is, +'Future division' is in effect). + +You can use scientific notation like ``1.3e-2`` to specify floating point +numbers. + +Here are scale functions equivalent to ``LINEAR(3, 3000)`` and +``LOG(3, 3000)``:: + + def scale_linear(f): + return 2997 * f + 3 + + def scale_log(f): + return exp(log(1000) * f) * 3 + + +Reporting +""""""""" + +Currently, there aren't any sophisticated reports. + +The standard report shows the candidates which have played most games; the +:mc-setting:`summary_spec` setting defines how many to show. + +In a line like:: + + (0,1) I: 0.01; F: 365.17 0.537 70 + +The ``(0,1)`` are the 'coordinates' of the candidate, ``I: 0.01; F: 365.17`` +are the engine parameters (identified using the :mc-setting:`format` setting), +``0.537`` is the win rate (including the :mc-setting:`initial_wins` and +:mc-setting:`initial_visits`), and ``70`` is the number of games (excluding +the :mc-setting:`initial_visits`). + +Also, after every :mc-setting:`log_tree_to_history_period` games, the status +of all candidates is written to the :ref:`history file ` (if +:mc-setting:`max_depth` > 1, the first two generations of candidates are +written). + + +.. _tree search: + +Tree search +""""""""""" + +As a further (and even more experimental) refinement, it's possible to arrange +the candidates in the form of a tree and use the :term:`UCT` algorithm instead +of plain :term:`UCB`. To do this, set the :mc-setting:`max_depth` setting to a +value greater than 1. + +Initially, this behaves as described in :ref:`The tuning algorithm `. But whenever a candidate is chosen for the second time, it +is :dfn:`expanded`: a new generation of candidates is created and placed as +that candidate's children in a tree structure. + +The new candidates are created by sampling their parent's 'division' of +optimiser parameter space in the same way as the full space was sampled to +make the first-generation candidates (so the number of children is again the +product of the :mc-setting:`split` settings). Their :math:`g_c` and :math:`w_c` +values are initialised to :mc-setting:`initial_visits` and +:mc-setting:`initial_wins` as usual. + +Then one of these child candidates is selected using the usual formula, where + +- :math:`g_c` is now the number of games the child has played + +- :math:`w_c` is now the number of games the child has won + +- :math:`g_p` is now the number of games the parent has played + +If :mc-setting:`max_depth` is greater than 2, then when a second-generation +candidate is chosen for the second time, it is expanded itself, and so on +until :mc-setting:`max_depth` is reached. + +Each time the tuner starts a new game, it walks down the tree using this +formula to choose a child node at each level, until it reaches a 'leaf' node. + +Once a candidate has been expanded, it does not play any further games; only +candidates which are 'leaf' nodes of the tree are used as players. The +:math:`g_c` and :math:`w_c` values for non-leaf candidates count the games and +wins played by the candidate's descendants, as well as by the candidate +itself. + +The 'best' candidate is determined by walking down the tree and choosing the +child with the most wins at each step (which may not end up with the leaf +candidate with the most wins in the entire tree). + + +.. note:: It isn't clear that using UCT for a continuous parameter space like + this is a wise (or valid) thing to do. I suspect it needs some form of RAVE + to perform well. + + +.. caution:: If you use a high :option:`--parallel ` + value, note that the Monte Carlo tuner doesn't currently take any action to + prevent the same unpromising branch of the tree being explored by multiple + processes simultaneously, which might lead to odd results (particularly if + you stop the competition and restart it). + + + + +Changing the control file between runs +"""""""""""""""""""""""""""""""""""""" + +In general, you shouldn't change the :mc-setting-cls:`Parameter` definitions +or the settings which control the tuning algorithm between runs. The +ringmaster will normally notice and refuse to start, but it's possible to fool +it and so get meaningless results. + +Changing the :mc-setting:`exploration_coefficient` is ok. Increasing +:mc-setting:`max_depth` is ok (decreasing it is ok too, but it won't stop the +tuner exploring parts of the tree that it has already expanded). + +Changing :mc-setting:`make_candidate` is ok, though if this affects player +behaviour it will probably be unhelpful. + +Changing :mc-setting:`initial_wins` or :mc-setting:`initial_visits` will have +no effect if :mc-setting:`max_depth` is 1; otherwise it will affect only +candidates created in future. + +Changing the settings which control reporting, including :mc-setting:`format`, +is ok. + +Changing :mc-setting:`number_of_games` is ok. + diff --git a/gomill/docs/playoffs.rst b/gomill/docs/playoffs.rst new file mode 100644 index 0000000..16da699 --- /dev/null +++ b/gomill/docs/playoffs.rst @@ -0,0 +1,214 @@ +.. index:: playoffs + +.. _playoff tournament: + +Playoff tournaments +^^^^^^^^^^^^^^^^^^^ + +:setting:`competition_type` string: ``"playoff"``. + +In a playoff tournament the control file explicitly describes one or more +pairings of players (:dfn:`matchups`). + +Each matchup is treated independently: different matchups can use different +board sizes, handicap arrangements, and so on. + +The tournament runs until :pl-setting:`number_of_games` have been played for +each matchup (indefinitely, if :pl-setting:`number_of_games` is unset). + + +.. contents:: Page contents + :local: + :backlinks: none + + +.. _sample_playoff_control_file: + +Sample control file +""""""""""""""""""" + +Here is a sample control file, illustrating how matchups are specified:: + + competition_type = 'playoff' + + players = { + 'gnugo-l1' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=1"), + + 'gnugo-l2' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=2"), + } + + board_size = 9 + komi = 6 + + matchups = [ + Matchup('gnugo-l1', 'gnugo-l2', board_size=13, + handicap=2, handicap_style='free', komi=0, + scorer='players', number_of_games=5), + + Matchup('gnugo-l1', 'gnugo-l2', alternating=True, + scorer='players', move_limit=200), + + Matchup('gnugo-l1', 'gnugo-l2', + komi=0.5, + scorer='internal'), + ] + + +.. _playoff_control_file_settings: + +Control file settings +""""""""""""""""""""" + +The following settings can be set at the top level of the control file: + +All :ref:`common settings `. + +All :ref:`game settings `, and the matchup settings +:pl-setting:`alternating` and :pl-setting:`number_of_games` described below; +these will be used for any matchups which don't explicitly override them. + +.. pl-setting:: matchups + + List of :pl-setting-cls:`Matchup` definitions (see :ref:`matchup + configuration`). + + This defines which players will compete against each other, and the game + settings they will use. + +The only required settings are :setting:`competition_type`, +:setting:`players`, and :pl-setting:`matchups`. + + + +.. _matchup configuration: + +Matchup configuration +""""""""""""""""""""" + +.. pl-setting-cls:: Matchup + +A :pl-setting-cls:`!Matchup` definition has the same syntax as a Python +function call: :samp:`Matchup({arguments})`. + +The first two arguments should be the :ref:`player codes ` for +the two players involved in the matchup. The remaining arguments should be +specified in keyword form. For example:: + + Matchup('gnugo-l1', 'fuego-5k', board_size=13, komi=6) + +Defaults for matchup arguments (other than :pl-setting:`id` and +:pl-setting:`name`) can be specified at the top level of the control file. + +The :setting:`board_size` and :setting:`komi` arguments must be given for all +matchups (either explicitly or as defaults); the rest are all optional. + +.. caution:: a default :setting:`komi` or :pl-setting:`alternating` setting + will be applied even to handicap games. + + +All :ref:`game settings ` can be used as matchup arguments, and +also the following: + + +.. _matchup id: + +.. pl-setting:: id + + Identifier + + A short string (usually one to three characters) which is used to identify + the matchup. Matchup ids appear in the :ref:`game ids ` (and so in + the |sgf| filenames), and are used in the :doc:`tournament results API + `. + + If this is left unspecified, the matchup id will be the index of the matchup + in the :pl-setting:`matchups` list (formatted as a decimal string, starting + from ``"0"``). + + +.. pl-setting:: name + + String + + A string used to describe the matchup in reports. By default, this has the + form :samp:`{player code} vs {player code}`; you may wish to change it if you + have more than one matchup between the same pair of players (perhaps with + different komi or handicap). + + +.. pl-setting:: alternating + + Boolean (default ``False``) + + If this is ``True``, the players will swap colours in successive games. + Otherwise, the player given as the first argument always takes Black. + + +.. pl-setting:: number_of_games + + Integer (default ``None``) + + The total number of games to play in the matchup. If you leave this unset, + there will be no limit. + + Changing :pl-setting:`!number_of_games` to ``0`` provides a way to effectively + disable a matchup in future runs, without forgetting its results. + + +Reporting +""""""""" + +The :ref:`live display ` and :ref:`competition report +` show each matchup's results in the following form:: + + gnugo-l1 v gnugo-l2 (5/5 games) + board size: 9 komi: 7.5 + wins black white avg cpu + gnugo-l1 2 40.00% 1 33.33% 1 50.00% 1.23 + gnugo-l2 3 60.00% 1 50.00% 2 66.67% 1.39 + 2 40.00% 3 60.00% + +or, if the players have not alternated colours:: + + gnugo-l1 v gnugo-l2 (5/5 games) + board size: 9 komi: 7.5 + wins avg cpu + gnugo-l1 0 0.00% (black) 0.49 + gnugo-l2 5 100.00% (white) 0.48 + +Any :term:`jigos ` are counted as half a win for each player. If any +games have been lost by forfeit, a count will be shown for each player. If any +games have unknown results (because they could not be scored, or reached the +:setting:`move_limit`), a count will be shown for each matchup. :ref:`void +games` are not shown in these reports. + +If there is more than one matchup between the same pair of players, use the +matchup :pl-setting:`name` setting to distinguish them. + + +Changing the control file between runs +"""""""""""""""""""""""""""""""""""""" + +If you change a :pl-setting-cls:`Matchup` definition, the new definition will +be used when describing the matchup in reports; there'll be no record of the +earlier definition, or which games were played under it. + +If you change a :pl-setting-cls:`Matchup` definition to have different players +(ie, player codes), the ringmaster will refuse to run the competition. + +If you delete a :pl-setting-cls:`Matchup` definition, results from that +matchup won't be displayed during future runs, but will be included (with some +missing information) in the :action:`report` and :action:`show` output. + +If you add a :pl-setting-cls:`Matchup` definition, put it at the end of the +list (or else explicitly specify the matchup ids). + +It's safe to increase or decrease a matchup's :pl-setting:`number_of_games`. +If more games have been played than the new limit, they will not be forgotten. + +In practice, you shouldn't delete :pl-setting-cls:`Matchup` definitions (if +you don't want any more games to be played, set :pl-setting:`number_of_games` +to ``0``). + diff --git a/gomill/docs/python-inv.txt b/gomill/docs/python-inv.txt new file mode 100644 index 0000000..25164ba --- /dev/null +++ b/gomill/docs/python-inv.txt @@ -0,0 +1,12 @@ +# Sphinx inventory version 1 +# Project: Python +# Version: 2.7 +os.path.expanduser function library/os.path.html +shlex.split function library/shlex.html +termios mod library/termios.html +math mod library/math.html +abs function library/functions.html +max function library/functions.html +min function library/functions.html +round function library/functions.html +datetime.date class library/datetime.html diff --git a/gomill/docs/results.rst b/gomill/docs/results.rst new file mode 100644 index 0000000..89db190 --- /dev/null +++ b/gomill/docs/results.rst @@ -0,0 +1,106 @@ +Viewing results +--------------- + +.. contents:: Page contents + :local: + :backlinks: none + +.. _competition report file: +.. index:: report file + +Reports +^^^^^^^ + +The competition :dfn:`report file` (:file:`{code}.report`) file is a plain +text description of the competition results. This is similar to the live +report that is displayed while the competition is running. It includes the +contents of the competition's :setting:`description` setting. + +For tournaments, it also shows descriptions of the players. These are obtained +using the |gtp| :gtp:`!name` and :gtp:`!version` commands, or using +:gtp:`gomill-describe_engine` if the engine provides it. + +For example, in a playoff tournament with a single matchup:: + + playoff: example + + Testing GNU Go level 1 vs level 2, 2010-10-14 + + gnugo-l1 v gnugo-l2 (5/5 games) + board size: 9 komi: 7.5 + wins black white avg cpu + gnugo-l1 2 40.00% 1 33.33% 1 50.00% 1.23 + gnugo-l2 3 60.00% 1 50.00% 2 66.67% 1.39 + 2 40.00% 3 60.00% + + player gnugo-l1: GNU Go:3.8 + player gnugo-l2: GNU Go:3.8 + + +The report file is written automatically at the end of each run. The +:action:`report` command line action forces it to be rewritten; this can be +useful if you have changed descriptive text in the control file, or if a run +stopped ungracefully. + +The :action:`show` command line action prints the same report to standard +output. + +It's safe to run :action:`show` or :action:`report` on a competition which is +currently being run. + + +.. _game records: + +Game records +^^^^^^^^^^^^ + +The ringmaster writes an |sgf| record of each game it plays to the +:file:`{code}.games/` directory (which it will create if necessary). This can +be disabled with the :setting:`record_games` setting. The filename is based on +the game's :ref:`game_id `. + +(You might also see game records in a :file:`{code}.void/` directory; these +are games which were abandoned due to software failure; see :ref:`void +games`.) + +The ringmaster supports a protocol for engines to provide text to be placed in +the comment section for individual moves: see :gtp:`gomill-explain_last_move`. + +The game record includes a description of the players in the root node comment +[#]_. If an engine implements :gtp:`gomill-describe_engine`, its output is +included. + +.. [#] The root node comment is used rather than the game comment because (in + my experience) |sgf| viewers tend to make it easier to see information + there. + + +.. index:: CPU time +.. index:: time; CPU + +.. _cpu time: + +CPU time +^^^^^^^^ + +The reports and game records show the CPU time taken by the players, when +available. + +If an engine implements the :gtp:`gomill-cpu_time` command, its output is +used. Otherwise, the ringmaster uses the CPU time of the engine process that +it created, as returned by the :c:func:`!wait4()` system call (user plus system +time); unfortunately, this may not be meaningful, if the engine's work isn't +all done directly in that process. + + +.. _querying the results: + +Querying the results +^^^^^^^^^^^^^^^^^^^^ + +Gomill provides a Python library interface for processing the game results +stored in a tournament's :ref:`state file `. + +This is documented in :doc:`tournament_results`. See also the +:script:`find_forfeits.py` example script. + diff --git a/gomill/docs/ringmaster.rst b/gomill/docs/ringmaster.rst new file mode 100644 index 0000000..f962b68 --- /dev/null +++ b/gomill/docs/ringmaster.rst @@ -0,0 +1,71 @@ +The ringmaster +============== + +The ringmaster is a command line program which arranges games between |gtp| +engines and keeps track of the results. See :doc:`ringmaster_cmdline` below +for details of the command line options. + +.. contents:: Page contents + :local: + :backlinks: none + +Competitions and runs +--------------------- + +.. index:: competition + +The ringmaster takes its instructions from a single configuration file known +as the :doc:`control file `. Each control file defines a +:term:`competition`; the :ref:`output files ` for that +competition are written to the directory containing the control file. + + +.. index:: run + +A competition can take place over multiple invocations of the ringmaster +(:dfn:`runs`). For example, a run can be halted from the console, in which +case starting the ringmaster again will make it continue from where it left +off. + + +Competition types +----------------- + +The ringmaster supports a number of different :dfn:`competition types`. +These are divided into :dfn:`tournaments` and :dfn:`tuning events`. + +In a tournament, the ringmaster plays games between predefined players, in +order to compare their strengths. The types of tournament are: + +Playoff tournaments + In a playoff tournament the control file explicitly describes one or more + pairings of players (:dfn:`matchups`). Each matchup can have separate + settings. + +All-play-all tournaments + In an all-play-all tournament the control file lists a number of players, and + games are played with the same settings between each possible pairing. + +In a tuning event, the ringmaster runs an algorithm for adjusting player +parameters to try to find the values which give strongest play. + +See :ref:`competition types` for full details of the types of tournament and +tuning event. + + + + +Using the ringmaster +-------------------- + +.. toctree:: + :maxdepth: 3 + :titlesonly: + + competitions + results + ringmaster_cmdline + settings + competition_types + Error handling… + diff --git a/gomill/docs/ringmaster_cmdline.rst b/gomill/docs/ringmaster_cmdline.rst new file mode 100644 index 0000000..f908c4d --- /dev/null +++ b/gomill/docs/ringmaster_cmdline.rst @@ -0,0 +1,80 @@ +Command line +^^^^^^^^^^^^ + +.. program:: ringmaster + +.. index:: action; ringmaster + +The ringmaster expects two command line arguments: the pathname of the control +file and an :dfn:`action`:: + + ringmaster [options] .ctl run + ringmaster [options] .ctl show + ringmaster [options] .ctl reset + ringmaster [options] .ctl check + ringmaster [options] .ctl report + ringmaster [options] .ctl stop + +The default action is :action:`!run`, so running a competition is normally a +simple line like:: + + $ ringmaster competitions/test.ctl + +See :ref:`Stopping competitions ` for the various ways +to stop the ringmaster. + + +The following actions are available: + +.. action:: run + + Starts the competition running. If the competition has been run previously, + it continues from where it left off. + +.. action:: show + + Prints a :ref:`report ` of the competition's + current status. This can be used for both running and stopped competitions. + +.. action:: reset + + Cleans up the competition completely. This deletes all output files, + including the competition's :ref:`state file `. + +.. action:: check + + Runs a test invocation of the competition's players. This is the same as the + :ref:`startup checks`, except that any output the players send to their + standard error stream will be printed. + +.. action:: report + + Rewrites the :ref:`competition report file ` based + on the current status. This can be used for both running and stopped + competitions. + +.. action:: stop + + Tells a running ringmaster for the competition to stop as soon as the + current games have completed. + + +The following options are available: + +.. option:: --parallel , -j + + Play N :ref:`simultaneous games `. + +.. option:: --quiet, -q + + Disable the on-screen reporting; see :ref:`Quiet mode `. + +.. option:: --max-games , -g + + Maximum number of games to play in the run; see :ref:`running + competitions`. + +.. option:: --log-gtp + + Log all |gtp| traffic; see :ref:`logging`. + diff --git a/gomill/docs/settings.rst b/gomill/docs/settings.rst new file mode 100644 index 0000000..a282de8 --- /dev/null +++ b/gomill/docs/settings.rst @@ -0,0 +1,534 @@ +.. _control file: + +The control file +---------------- + +.. contents:: Page contents + :local: + :backlinks: none + + +.. _sample control file: + +Sample control file +^^^^^^^^^^^^^^^^^^^ + +Here is a sample control file for a playoff tournament:: + + competition_type = 'playoff' + + description = """ + This is a sample control file. + + It illustrates player definitions, common settings, and game settings. + """ + + record_games = True + stderr_to_log = False + + players = { + # GNU Go level 1 + 'gnugo-l1' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=1"), + + # GNU Go level 2 + 'gnugo-l2' : Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=2"), + + # Fuego at 5000 playouts per move + 'fuego-5k' : Player("fuego --quiet", + startup_gtp_commands=[ + "go_param timelimit 999999", + "uct_max_memory 350000000", + "uct_param_search number_threads 1", + "uct_param_player reuse_subtree 0", + "uct_param_player ponder 0", + "uct_param_player max_games 5000", + ]), + } + + board_size = 9 + komi = 6 + + matchups = [ + Matchup('gnugo-l1', 'fuego-5k', board_size=13, + handicap=2, handicap_style='free', komi=0, + scorer='players', number_of_games=5), + + Matchup('gnugo-l2', 'fuego-5k', alternating=True, + scorer='players', move_limit=200), + + Matchup('gnugo-l1', 'gnugo-l2', + komi=0.5, + scorer='internal'), + ] + + +File format +^^^^^^^^^^^ + +The control file is a plain text configuration file. + +It is interpreted in the same way as a Python source file. See the +:ref:`sample control file` above for an example of the syntax. + + .. __: http://docs.python.org/release/2.7/reference/index.html + +The control file is made up of a series of top-level :dfn:`settings`, in the +form of assignment statements: :samp:`{setting_name} = {value}`. + +Each top-level setting should begin on a new line, in the leftmost column of +the file. Settings which use brackets of any kind can be split over multiple +lines between elements (for example, lists can be split at the commas). + +Comments are introduced by the ``#`` character, and continue until the end of +the line. + +In general, the order of settings in the control file isn't significant +(except for list members). But note that :setting:`competition_type` must come +first. + +See :ref:`data types` below for the representation of values. See the `Python +language reference`__ for a formal specification. + +The settings which are common to all competition types are listed below. +Further settings are given on the page documenting each competition type. + +.. caution:: while the ringmaster will give error messages for unacceptable + setting values, it will ignore attempts to set a nonexistent setting (this + is because you're allowed to define variables of your own in the control + file and use them in later setting definitions). + +If you wish, you can use arbitrary Python expressions in the control file; see +:ref:`control file techniques` below. + +.. caution:: all Python code in the control file will be executed; a hostile + party with write access to a control file can cause the ringmaster to + execute arbitrary code. On a shared system, do not make the competition + directory or the control file world-writeable. + +The recommended filename extension for the control file is :file:`.ctl`. + +.. _data types: + +Data types +^^^^^^^^^^ + +The following data types are used for values of settings: + +String + A literal string of characters in single or double quotes, eg ``'gnugo-l1'`` + or ``"free"``. + + Strings containing non-ASCII characters should be encoded as UTF-8 (Python + unicode objects are also accepted). + + Strings can be broken over multiple lines by writing adjacent literals + separated only by whitespace; see the :setting-cls:`Player` definitions in + the example above. + + Backslash escapes can be used in strings, such as ``\n`` for a newline. + Alternatively, three (single or double) quotes can be used for a multi-line + string; see ``description`` in the example above. + +Identifier + A (short) string made up of any combination of ASCII letters, numerals, and + the punctuation characters ``- ! $ % & * + - . : ; < = > ? ^ _ ~``. + +Boolean + A truth value, written as ``True`` or ``False``. + +Integer + A whole number, written as a decimal literal, eg ``19`` or ``-1``. + +Float + A floating-point number, written as a decimal literal, eg ``6`` or ``6.0`` + or ``6.5``. + +List + A sequence of values of uniform type, written with square brackets separated + by commas, eg ``["max_playouts 3000", "initial_wins 5"]``. An extra comma + after the last item is harmless. + +Dictionary + An explicit map of keys of uniform type to values of uniform type, written + with curly brackets, colons, and commas, eg ``{'p1' : True, 'p2' : False}``. + An extra comma after the last item is harmless. + + +.. _file and directory names: + +File and directory names +^^^^^^^^^^^^^^^^^^^^^^^^ + +When values in the control file are file or directory names, non-absolute +names are interpreted relative to the :ref:`competition directory `. + +If a file or directory name begins with ``~``, home directory expansion is +applied (see :func:`os.path.expanduser`). + + +.. _common settings: + +Common settings +^^^^^^^^^^^^^^^ + +The following settings can appear at the top level of the control file for all +competition types. + +.. setting:: competition_type + + String: ``"playoff"``, ``"allplayall"``, ``"mc_tuner"``, or ``"ce_tuner"`` + + Determines the type of tournament or tuning event. This must be set on the + first line in the control file (not counting blank lines and comments). + + +.. setting:: description + + String (default ``None``) + + A text description of the competition. This will be included in the + :ref:`competition report file `. Leading and + trailing whitespace is ignored. + + +.. setting:: record_games + + Boolean (default ``True``) + + Write |sgf| :ref:`game records `. + + +.. setting:: stderr_to_log + + Boolean (default ``True``) + + Redirect all players' standard error streams to the :ref:`event log + `. See :ref:`standard error`. + + +.. _player codes: + +.. index:: player code + +.. setting:: players + + Dictionary mapping identifiers to :setting-cls:`Player` definitions (see + :ref:`player configuration`). + + Describes the |gtp| engines that can be used in the competition. If you wish + to use the same program with different settings, each combination of + settings must be given its own :setting-cls:`Player` definition. See + :ref:`control file techniques` below for a compact way to define several + similar Players. + + The dictionary keys are the :dfn:`player codes`; they are used to identify + the players in reports and the |sgf| game records, and elsewhere in the + control file to specify how players take part in the competition. + + See the pages for specific competition types for the way in which players + are selected from the :setting:`!players` dictionary. + + It's fine to have player definitions here which aren't used in the + competition. These definitions will be ignored, and no corresponding engines + will be run. + + + +.. _player configuration: + +Player configuration +^^^^^^^^^^^^^^^^^^^^ + +.. setting-cls:: Player + +A :setting-cls:`!Player` definition has the same syntax as a Python function +call: :samp:`Player({arguments})`. Apart from :setting:`command`, the +arguments should be specified using keyword form (see the examples for +particular arguments below). + +All arguments other than :setting:`command` are optional. + +.. tip:: For results to be meaningful, you should normally configure players + to use a fixed amount of computing power, paying no attention to the amount + of real time that passes, and make sure :term:`pondering` is not turned on. + +The arguments are: + + +.. setting:: command + + String or list of strings + + This is the only required :setting-cls:`Player` argument. It can be + specified either as the first argument, or using a keyword + :samp:`command="{...}"`. It specifies the executable which will provide the + player, and its command line arguments. + + The player subprocess is executed directly, not run via a shell. + + The :setting:`!command` can be either a string or a list of strings. If it + is a string, it is split using rules similar to a Unix shell's (see + :func:`shlex.split`). + + In either case, the first element is taken as the executable name and the + remainder as its arguments. + + If the executable name does not contain a ``/``, it is searched for on the + the :envvar:`!PATH`. Otherwise it is handled as described in :ref:`file and + directory names `. + + Example:: + + Player("~/src/fuego-svn/fuegomain/fuego --quiet") + + +.. setting:: cwd + + String (default ``None``) + + The working directory for the player. + + If this is left unset, the player's working directory will be the working + directory from when the ringmaster was launched (which may not be the + competition directory). Use ``cwd="."`` to specify the competition + directory. + + .. tip:: + If an engine writes debugging information to its working directory, use + :setting:`cwd` to get it out of the way:: + + Player('mogo', cwd='~/tmp') + + +.. setting:: environ + + Dictionary mapping strings to strings (default ``None``) + + This specifies environment variables to be set in the player process, in + addition to (or overriding) those inherited from its parent. + + Note that there is no special handling in this case for values which happen + to be file or directory names. + + Example:: + + Player('goplayer', environ={'GOPLAYER-DEBUG' : 'true'}) + + +.. setting:: discard_stderr + + Boolean (default ``False``) + + Redirect the player's standard error stream to :file:`/dev/null`. See + :ref:`standard error`. + + Example:: + + Player('mogo', discard_stderr=True) + + +.. setting:: startup_gtp_commands + + List of strings, or list of lists of strings (default ``None``) + + |gtp| commands to send at the beginning of each game. See :ref:`playing + games`. + + Each command can be specified either as a single string or as a list of + strings (with each |gtp| argument in a single string). For example, the + following are equivalent:: + + Player('fuego', startup_gtp_commands=[ + "uct_param_player ponder 0", + "uct_param_player max_games 5000"]) + + Player('fuego', startup_gtp_commands=[ + ["uct_param_player", "ponder", "0"], + ["uct_param_player", "max_games", "5000"]]) + + +.. setting:: gtp_aliases + + Dictionary mapping strings to strings (default ``None``) + + This is a map of |gtp| command names to command names, eg:: + + Player('fuego', gtp_aliases={'gomill-cpu_time' : 'cputime'}) + + When the ringmaster would normally send :gtp:`gomill-cpu_time`, it will send + :gtp:`!cputime` instead. + + The command names are case-sensitive. There is no mechanism for altering + arguments. + + +.. setting:: is_reliable_scorer + + Boolean (default ``True``) + + If the :setting:`scorer` setting is ``players``, the ringmaster normally + asks each player that implements the :gtp:`!final_score` |gtp| command to + report the game result. Setting :setting:`!is_reliable_scorer` to ``False`` + for a player causes that player never to be asked. + + +.. setting:: allow_claim + + Boolean (default ``False``) + + Permits the player to claim a win (using the |gtp| extension + :gtp:`gomill-genmove_ex`). See :ref:`claiming wins`. + + +.. _game settings: + +Game settings +^^^^^^^^^^^^^ + +The following settings describe how a particular game is to be played. + +They are not all used in every competition type, and may be specified in some +other way than a top level control file setting; see the page documenting a +particular competition type for details. + + +.. setting:: board_size + + Integer + + The size of Go board to use for the game (eg ``19`` for a 19x19 game). The + ringmaster is willing to use board sizes from 2 to 25. + + +.. setting:: komi + + Float + + The :term:`komi` to use for the game. You can specify any floating-point + value, and it will be passed on to the |gtp| engines unchanged, but normally + only integer or half-integer values will be useful. Negative values are + allowed. + + +.. setting:: handicap + + Integer (default ``None``) + + The number of handicap stones to give Black at the start of the game. See + also :setting:`handicap_style`. + + See the `GTP specification`_ for the rules about what handicap values + are permitted for different board sizes (in particular, values less than 2 + are never allowed). + + +.. setting:: handicap_style + + String: ``"fixed"`` or ``"free"`` (default ``"fixed"``) + + Determines whether the handicap stones are placed on prespecified points, or + chosen by the Black player. See the `GTP specification`_ for more details. + + This is ignored if :setting:`handicap` is unset. + + .. _GTP specification: http://www.lysator.liu.se/~gunnar/gtp/gtp2-spec-draft2/gtp2-spec.html#SECTION00051000000000000000 + + + +.. setting:: move_limit + + Integer (default ``1000``) + + The maximum number of moves to allow in a game. If this limit is reached, + the game is stopped; see :ref:`playing games`. + + +.. setting:: scorer + + String: ``"players"`` or ``"internal"`` (default ``"players"``) + + Determines whether the game result is determined by the engines, or by the + ringmaster. See :ref:`Scoring ` and :setting:`is_reliable_scorer`. + + +.. setting:: internal_scorer_handicap_compensation + + String: ``"no"``, ``"full"`` or ``"short"`` (default ``"full"``) + + Specifies whether White is given extra points to compensate for Black's + handicap stones; see :ref:`Scoring ` for details. This setting has + no effect for games which are played without handicap, and it has no effect + when :setting:`scorer` is set to ``"players"``. + + + + + +Changing the control file between runs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Changing the control file between runs of the same competition (or after the +final run) is allowed. For example, in a playoff tournament it's fine to +increase a completed matchup's :pl-setting:`number_of_games` and set the +competition off again. + +The intention is that nothing surprising should happen if you change the +control file; of course if you change settings which affect player behaviour +then result summaries might not be meaningful. + +In particular, if you change a :setting-cls:`Player` definition, the new +definition will be used when describing the player in reports; there'll be no +record of the earlier definition, or which games were played under it. + +If you change descriptive text, you can use the :action:`report` command line +action to remake the report file. + +The page documenting each competition type has more detail on what it is safe +to change. + + +.. _control file techniques: + +Control file techniques +^^^^^^^^^^^^^^^^^^^^^^^ + +As the control file is just Python code, it's possible to use less direct +methods to specify the values of settings. + +One convenient way to define a number of similar players is to define a +function which returns a :setting-cls:`Player` object. For example, the player +definitions in the sample control file could be rewritten as follows:: + + def gnugo(level): + return Player("gnugo --mode=gtp --chinese-rules " + "--capture-all-dead --level=%d" % level) + + def fuego(playouts_per_move, additional_commands=[]): + commands = [ + "go_param timelimit 999999", + "uct_max_memory 350000000", + "uct_param_search number_threads 1", + "uct_param_player reuse_subtree 0", + "uct_param_player ponder 0", + "uct_param_player max_games %d" % playouts_per_move, + ] + return Player( + "fuego --quiet", + startup_gtp_commands=commands+additional_commands) + + players = { + 'gnugo-l1' : gnugo(level=1), + 'gnugo-l2' : gnugo(level=2), + 'fuego-5k' : fuego(playouts_per_move=5000) + } + +If you assign to a setting more than once, the final value is the one that +counts. Settings specified above as having default ``None`` can be assigned +the value ``None``, which will be equivalent to leaving them unset. + +Importing parts of the Python standard library (or other Python libraries that +you have installed) is allowed. + diff --git a/gomill/docs/sgf.rst b/gomill/docs/sgf.rst new file mode 100644 index 0000000..455ed11 --- /dev/null +++ b/gomill/docs/sgf.rst @@ -0,0 +1,914 @@ +SGF support +----------- + +.. module:: gomill.sgf + :synopsis: High level SGF interface. + +.. versionadded:: 0.7 + +Gomill's |sgf| support is intended for use with version FF[4], which is +specified at http://www.red-bean.com/sgf/index.html. It has support for the +game-specific properties for Go, but not those of other games. Point, Move and +Stone values are interpreted as Go points. + +The :mod:`gomill.sgf` module provides the main support. This module is +independent of the rest of Gomill. + +The :mod:`gomill.sgf_moves` module contains some higher-level functions for +processing moves and positions, and provides a link to the +:mod:`.boards` module. + +The :mod:`!gomill.sgf_grammar` and :mod:`!gomill.sgf_properties` modules are +used to implement the :mod:`!sgf` module, and are not currently documented. + + +.. contents:: Page contents + :local: + :backlinks: none + +Examples +^^^^^^^^ + +Reading and writing:: + + >>> from gomill import sgf + >>> g = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9];B[ee];W[ge])") + >>> g.get_size() + 9 + >>> root_node = g.get_root() + >>> root_node.get("SZ") + 9 + >>> root_node.get_raw("SZ") + '9' + >>> root_node.set("RE", "B+R") + >>> new_node = g.extend_main_sequence() + >>> new_node.set_move("b", (2, 3)) + >>> [node.get_move() for node in g.get_main_sequence()] + [(None, None), ('b', (4, 4)), ('w', (4, 6)), ('b', (2, 3))] + >>> g.serialise() + '(;FF[4]GM[1]RE[B+R]SZ[9];B[ee];W[ge];B[dg])\n' + + +Recording a game:: + + g = sgf.Sgf_game(size=13) + for move_info in ...: + node = g.extend_main_sequence() + node.set_move(move_info.colour, move_info.move) + if move_info.comment is not None: + node.set("C", move_info.comment) + with open(pathname, "w") as f: + f.write(g.serialise()) + +See also the :script:`show_sgf.py` and :script:`split_sgf_collection.py` +example scripts. + + +Sgf_game objects +^^^^^^^^^^^^^^^^ + +|sgf| data is represented using :class:`!Sgf_game` objects. Each object +represents the data for a single |sgf| file (corresponding to a ``GameTree`` +in the |sgf| spec). This is typically used to represent a single game, +possibly with variations (but it could be something else, such as a problem +set). + +An :class:`!Sgf_game` can either be created from scratch or loaded from a +string. + +To create one from scratch, instantiate an :class:`!Sgf_game` object directly: + +.. class:: Sgf_game(size, encoding="UTF-8"]) + + *size* is an integer from 1 to 26, indicating the board size. + + The optional *encoding* parameter specifies the :ref:`raw property encoding + ` to use for the game. + +When a game is created this way, the following root properties are initially +set: :samp:`FF[4]`, :samp:`GM[1]`, :samp:`SZ[{size}]`, and +:samp:`CA[{encoding}]`. + +To create a game from existing |sgf| data, use the +:func:`!Sgf_game.from_string` classmethod: + +.. classmethod:: Sgf_game.from_string(s[, override_encoding=None]) + + :rtype: :class:`!Sgf_game` + + Creates an :class:`!Sgf_game` from the |sgf| data in *s*, which must be an + 8-bit string. + + The board size and :ref:`raw property encoding ` are + taken from the ``SZ`` and ``CA`` properties in the root node (defaulting to + ``19`` and ``"ISO-8859-1"``, respectively). Board sizes greater than ``26`` + are rejected. + + If *override_encoding* is present, the source data is assumed to be in the + encoding it specifies (no matter what the ``CA`` property says), and the + ``CA`` property and raw property encoding are changed to match. + + Raises :exc:`ValueError` if it can't parse the string, or if the ``SZ`` or + ``CA`` properties are unacceptable. No error is reported for other + malformed property values. See also :ref:`parsing_details` below. + + Example:: + + g = sgf.Sgf_game.from_string( + "(;FF[4]GM[1]SZ[9]CA[UTF-8];B[ee];W[ge])", + override_encoding="iso8859-1") + + +To retrieve the |sgf| data as a string, use the :meth:`!serialise` method: + +.. method:: Sgf_game.serialise([wrap]) + + :rtype: string + + Produces the |sgf| representation of the data in the :class:`!Sgf_game`. + + Returns an 8-bit string, in the encoding specified by the ``CA`` root + property (defaulting to ``"ISO-8859-1"``). + + See :ref:`transcoding ` below for details of the behaviour if + the ``CA`` property is changed from its initial value. + + This makes some effort to keep the output line length to no more than 79 + bytes. Pass ``None`` in the *wrap* parameter to disable this behaviour, or + pass an integer to specify a different limit. + + +The complete game tree is represented using :class:`Tree_node` objects, which +are used to access the |sgf| properties. An :class:`!Sgf_game` always has at +least one node, the :dfn:`root node`. + +.. method:: Sgf_game.get_root() + + :rtype: :class:`Tree_node` + + Returns the root node of the game tree. + +The root node contains global properties for the game tree, and typically also +contains *game-info* properties. It sometimes also contains *setup* properties +(for example, if the game does not begin with an empty board). + +Changing the ``FF`` and ``GM`` properties is permitted, but Gomill will carry +on using the FF[4] and GM[1] (Go) rules. Changing ``SZ`` is not permitted (but +if the size is 19 you may remove the property). Changing ``CA`` is permitted +(this controls the encoding used by :meth:`~Sgf_game.serialise`). + + +.. rubric:: Convenience methods for tree access + +The complete game tree can be accessed through the root node, but the +following convenience methods are also provided. They return the same +:class:`Tree_node` objects that would be reached via the root node. + +Some of the convenience methods are for accessing the :dfn:`leftmost` +variation of the game tree. This is the variation which appears first in the +|sgf| ``GameTree``, often shown in graphical editors as the topmost horizontal +line of nodes. In a game tree without variations, the leftmost variation is +just the whole game. + + +.. method:: Sgf_game.get_last_node() + + :rtype: :class:`Tree_node` + + Returns the last (leaf) node in the leftmost variation. + +.. method:: Sgf_game.get_main_sequence() + + :rtype: list of :class:`Tree_node` objects + + Returns the complete leftmost variation. The first element is the root + node, and the last is a leaf. + +.. method:: Sgf_game.get_main_sequence_below(node) + + :rtype: list of :class:`Tree_node` objects + + Returns the leftmost variation beneath the :class:`Tree_node` *node*. The + first element is the first child of *node*, and the last is a leaf. + + Note that this isn't necessarily part of the leftmost variation of the + game as a whole. + +.. method:: Sgf_game.get_main_sequence_above(node) + + :rtype: list of :class:`Tree_node` objects + + Returns the partial variation leading to the :class:`Tree_node` *node*. The + first element is the root node, and the last is the parent of *node*. + +.. method:: Sgf_game.extend_main_sequence() + + :rtype: :class:`Tree_node` + + Creates a new :class:`Tree_node`, adds it to the leftmost variation, and + returns it. + + This is equivalent to + :meth:`get_last_node`\ .\ :meth:`~Tree_node.new_child` + + +.. rubric:: Convenience methods for root properties + +The following methods provide convenient access to some of the root node's +|sgf| properties. The main difference between using these methods and using +:meth:`~Tree_node.get` on the root node is that these methods return the +appropriate default value if the property is not present. + +.. method:: Sgf_game.get_size() + + :rtype: integer + + Returns the board size (``19`` if the ``SZ`` root property isn't present). + +.. method:: Sgf_game.get_charset() + + :rtype: string + + Returns the effective value of the ``CA`` root property (``ISO-8859-1`` if + the ``CA`` root property isn't present). + + The returned value is a codec name in normalised form, which may not be + identical to the string returned by ``get_root().get("CA")``. Raises + :exc:`ValueError` if the property value doesn't identify a Python codec. + + This gives the encoding that would be used by :meth:`serialise`. It is not + necessarily the same as the :ref:`raw property encoding + ` (use :meth:`~Tree_node.get_encoding` on the root + node to retrieve that). + + +.. method:: Sgf_game.get_komi() + + :rtype: float + + Returns the :term:`komi` (``0.0`` if the ``KM`` root property isn't + present). + + Raises :exc:`ValueError` if the ``KM`` root property is present but + malformed. + +.. method:: Sgf_game.get_handicap() + + :rtype: integer or ``None`` + + Returns the number of handicap stones. + + Returns ``None`` if the ``HA`` root property isn't present, or if it has + value zero (which isn't strictly permitted). + + Raises :exc:`ValueError` if the ``HA`` property is otherwise malformed. + +.. method:: Sgf_game.get_player_name(colour) + + :rtype: string or ``None`` + + Returns the name of the specified player, or ``None`` if the required + ``PB`` or ``PW`` root property isn't present. + +.. method:: Sgf_game.get_winner() + + :rtype: *colour* + + Returns the colour of the winning player. + + Returns ``None`` if the ``RE`` root property isn't present, or if neither + player won. + +.. method:: Sgf_game.set_date([date]) + + Sets the ``DT`` root property, to a single date. + + If *date* is specified, it should be a :class:`datetime.date`. Otherwise + the current date is used. + + (|sgf| allows ``DT`` to be rather more complicated than a single date, so + there's no corresponding get_date() method.) + + +Tree_node objects +^^^^^^^^^^^^^^^^^ + +.. class:: Tree_node + + A Tree_node object represents a single node from an |sgf| file. + + Don't instantiate Tree_node objects directly; retrieve them from + :class:`Sgf_game` objects. + + Tree_node objects have the following attributes (which should be treated as + read-only): + + .. attribute:: owner + + The :class:`Sgf_game` that the node belongs to. + + .. attribute:: parent + + The node's parent :class:`!Tree_node` (``None`` for the root node). + + +.. rubric:: Tree navigation + +A :class:`!Tree_node` acts as a list-like container of its children: it can be +indexed, sliced, and iterated over like a list, and it supports the `index`__ +method. A :class:`!Tree_node` with no children is treated as having truth +value false. For example, to find all leaf nodes:: + + def print_leaf_comments(node): + if node: + for child in node: + print_leaf_comments(child) + else: + if node.has_property("C"): + print node.get("C") + else: + print "--" + +.. __: http://docs.python.org/release/2.7/library/stdtypes.html#mutable-sequence-types + + +.. rubric:: Property access + +Each node holds a number of :dfn:`properties`. Each property is identified by +a short string called the :dfn:`PropIdent`, eg ``"SZ"`` or ``"B"``. See +:ref:`sgf_property_list` below for a list of the standard properties. See the +:term:`SGF` specification for full details. See :ref:`parsing_details` below +for restrictions on well-formed *PropIdents*. + +Gomill doesn't enforce |sgf|'s restrictions on where properties can appear +(eg, the distinction between *setup* and *move* properties). + +The principal methods for accessing the node's properties are: + +.. method:: Tree_node.get(identifier) + + Returns a native Python representation of the value of the property whose + *PropIdent* is *identifier*. + + Raises :exc:`KeyError` if the property isn't present. + + Raises :exc:`ValueError` if it detects that the property value is + malformed. + + See :ref:`sgf_property_types` below for details of how property values are + represented in Python. + + See :ref:`sgf_property_list` below for a list of the known properties. + Setting nonstandard properties is permitted; they are treated as having + type Text. + +.. method:: Tree_node.set(identifier, value) + + Sets the value of the property whose *PropIdent* is *identifier*. + + *value* should be a native Python representation of the required property + value (as returned by :meth:`get`). + + Raises :exc:`ValueError` if the property value isn't acceptable. + + See :ref:`sgf_property_types` below for details of how property values + should be represented in Python. + + See :ref:`sgf_property_list` below for a list of the known properties. Any + other property is treated as having type Text. + +.. method:: Tree_node.unset(identifier) + + Removes the property whose *PropIdent* is *identifier* from the node. + + Raises :exc:`KeyError` if the property isn't currently present. + +.. method:: Tree_node.has_property(identifier) + + :rtype: bool + + Checks whether the property whose *PropIdent* is *identifier* is present. + +.. method:: Tree_node.properties() + + :rtype: list of strings + + Lists the properties which are present in the node. + + Returns a list of *PropIdents*, in unspecified order. + +.. method:: Tree_node.find_property(identifier) + + Returns the value of the property whose *PropIdent* is *identifier*, + looking in the node's ancestors if necessary. + + This is intended for use with properties of type *game-info*, and with + properties which have the *inherit* attribute. + + It looks first in the node itself, then in its parent, and so on up to the + root, returning the first value it finds. Otherwise the behaviour is the + same as :meth:`get`. + + Raises :exc:`KeyError` if no node defining the property is found. + + +.. method:: Tree_node.find(identifier) + + :rtype: :class:`!Tree_node` or ``None`` + + Returns the nearest node defining the property whose *PropIdent* is + *identifier*. + + Searches in the same way as :meth:`find_property`, but returns the node + rather than the property value. Returns ``None`` if no node defining the + property is found. + + +.. rubric:: Convenience methods for properties + +The following convenience methods are also provided, for more flexible access +to a few of the most important properties: + +.. method:: Tree_node.get_move() + + :rtype: tuple (*colour*, *move*) + + Indicates which of the the ``B`` or ``W`` properties is present, and + returns its value. + + Returns (``None``, ``None``) if neither property is present. + +.. method:: Tree_node.set_move(colour, move) + + Sets the ``B`` or ``W`` property. If the other property is currently + present, it is removed. + + Gomill doesn't attempt to ensure that moves are legal. + +.. method:: Tree_node.get_setup_stones() + + :rtype: tuple (set of *points*, set of *points*, set of *points*) + + Returns the settings of the ``AB``, ``AW``, and ``AE`` properties. + + The tuple elements represent black, white, and empty points respectively. + If a property is missing, the corresponding set is empty. + +.. method:: Tree_node.set_setup_stones(black, white[, empty]) + + Sets the ``AB``, ``AW``, and ``AE`` properties. + + Each parameter should be a sequence or set of *points*. If a parameter + value is empty (or, in the case of *empty*, if the parameter is + omitted) the corresponding property will be unset. + +.. method:: Tree_node.has_setup_stones() + + :rtype: bool + + Returns ``True`` if the ``AB``, ``AW``, or ``AE`` property is present. + +.. method:: Tree_node.add_comment_text(text) + + If the ``C`` property isn't already present, adds it with the value given + by the string *text*. + + Otherwise, appends *text* to the existing ``C`` property value, preceded by + two newlines. + + +.. rubric:: Board size and raw property encoding + +Each :class:`!Tree_node` knows its game's board size, and its :ref:`raw +property encoding ` (because these are needed to +interpret property values). They can be retrieved using the following methods: + +.. method:: Tree_node.get_size() + + :rtype: int + +.. method:: Tree_node.get_encoding() + + :rtype: string + + This returns the name of the raw property encoding (in a normalised form, + which may not be the same as the string originally used to specify the + encoding). + +An attempt to change the value of the ``SZ`` property so that it doesn't match +the board size will raise :exc:`ValueError` (even if the node isn't the root). + + +.. rubric:: Access to raw property values + +Raw property values are 8-bit strings, containing the exact bytes that go +between the ``[`` and ``]`` in the |sgf| file. They should be treated as being +encoded in the node's :ref:`raw property encoding ` +(but there is no guarantee that they hold properly encoded data). + +The following methods are provided for access to raw property values. They can +be used to access malformed values, or to avoid the standard escape processing +and whitespace conversion for Text and SimpleText values. + +When setting raw property values, any string that is a well formed |sgf| +*PropValue* is accepted: that is, any string that that doesn't contain an +unescaped ``]`` or end with an unescaped ``\``. There is no check that the +string is properly encoded in the raw property encoding. + +.. method:: Tree_node.get_raw_list(identifier) + + :rtype: nonempty list of 8-bit strings + + Returns the raw values of the property whose *PropIdent* is *identifier*. + + Raises :exc:`KeyError` if the property isn't currently present. + + If the property value is an empty elist, returns a list containing a single + empty string. + +.. method:: Tree_node.get_raw(identifier) + + :rtype: 8-bit string + + Returns the raw value of the property whose *PropIdent* is *identifier*. + + Raises :exc:`KeyError` if the property isn't currently present. + + If the property has multiple `PropValue`\ s, returns the first. If the + property value is an empty elist, returns an empty string. + +.. method:: Tree_node.get_raw_property_map(identifier) + + :rtype: dict: string → list of 8-bit strings + + Returns a dict mapping *PropIdents* to lists of raw values. + + Returns the same dict object each time it's called. + + Treat the returned dict object as read-only. + +.. method:: Tree_node.set_raw_list(identifier, values) + + Sets the raw values of the property whose *PropIdent* is *identifier*. + + *values* must be a nonempty list of 8-bit strings. To specify an empty + elist, pass a list containing a single empty string. + + Raises :exc:`ValueError` if the identifier isn't a well-formed *PropIdent*, + or if any value isn't a well-formed *PropValue*. + +.. method:: Tree_node.set_raw(identifier, value) + + Sets the raw value of the property whose *PropIdent* is *identifier*. + + Raises :exc:`ValueError` if the identifier isn't a well-formed *PropIdent*, + or if the value isn't a well-formed *PropValue*. + + +.. rubric:: Tree manipulation + +The following methods are provided for manipulating the tree: + +.. method:: Tree_node.new_child([index]) + + :rtype: :class:`!Tree_node` + + Creates a new :class:`!Tree_node` and adds it to the tree as this node's + last child. + + If the optional integer *index* parameter is present, the new node is + inserted in the list of children at the specified index instead (with the + same behaviour as :meth:`!list.insert`). + + Returns the new node. + +.. method:: Tree_node.delete() + + Removes the node from the tree (along with all its descendents). + + Raises :exc:`ValueError` if called on the root node. + + You should not continue to use a node which has been removed from its tree. + +.. method:: Tree_node.reparent(new_parent[, index]) + + Moves the node from one part of the tree to another (along with all its + descendents). + + *new_parent* must be a node belonging to the same game. + + Raises :exc:`ValueError` if the operation would create a loop in the tree + (ie, if *new_parent* is the node being moved or one of its descendents). + + If the optional integer *index* parameter is present, the new node is + inserted in the new parent's list of children at the specified index; + otherwise it is placed at the end. + + This method can be used to reorder variations. For example, to make a node + the leftmost variation of its parent:: + + node.reparent(node.parent, 0) + + +.. _sgf_property_types: + +Property types +^^^^^^^^^^^^^^ + +The :meth:`~Tree_node.get` and :meth:`~Tree_node.set` node methods convert +between raw |sgf| property values and suitable native Python types. + +The following table shows how |sgf| property types are represented as Python +values: + +=========== ======================== +|sgf| type Python representation +=========== ======================== +None ``True`` +Number int +Real float +Double ``1`` or ``2`` (int) +Colour *colour* +SimpleText 8-bit UTF-8 string +Text 8-bit UTF-8 string +Stone *point* +Point *point* +Move *move* +=========== ======================== + +Gomill doesn't distinguish the Point and Stone |sgf| property types. It +rejects representations of 'pass' for the Point and Stone types, but accepts +them for Move (this is not what is described in the |sgf| specification, but +it does correspond to the properties in which 'pass' makes sense). + +Values of list or elist types are represented as Python lists. An empty elist +is represented as an empty Python list (in contrast, the raw value is a list +containing a single empty string). + +Values of compose types are represented as Python pairs (tuples of length +two). ``FG`` values are either a pair (int, string) or ``None``. + +For Text and SimpleText values, :meth:`~Tree_node.get` and +:meth:`~Tree_node.set` take care of escaping. You can store arbitrary strings +in a Text value and retrieve them unchanged, with the following exceptions: + +* all linebreaks are are normalised to ``\n`` + +* whitespace other than line breaks is converted to a single space + +:meth:`~Tree_node.get` accepts compressed point lists, but +:meth:`~Tree_node.set` never produces them (some |sgf| viewers still don't +support them). + +In some cases, :meth:`~Tree_node.get` will accept values which are not +strictly permitted in |sgf|, if there's a sensible way to interpret them. In +particular, empty lists are accepted for all list types (not only elists). + +In some cases, :meth:`~Tree_node.set` will accept values which are not exactly +in the Python representation listed, if there's a natural way to convert them +to the |sgf| representation. + +Both :meth:`~Tree_node.get` and :meth:`~Tree_node.set` check that Point values +are in range for the board size. Neither :meth:`~Tree_node.get` nor +:meth:`~Tree_node.set` pays attention to range restrictions for values of type +Number. + +Examples:: + + >>> node.set('KO', True) + >>> node.get_raw('KO') + '' + >>> node.set('HA', 3) + >>> node.set('KM', 5.5) + >>> node.set('GB', 2) + >>> node.set('PL', 'w') + >>> node.set('RE', 'W+R') + >>> node.set('GC', 'Example game\n[for documentation]') + >>> node.get_raw('GC') + 'Example game\n[for documentation\\]' + >>> node.set('B', (2, 3)) + >>> node.get_raw('B') + 'dg' + >>> node.set('LB', [((6, 0), "label 1"), ((6, 1), "label 2")]) + >>> node.get_raw_list('LB') + ['ac:label 1', 'bc:label 2'] + + + +.. _sgf_property_list: + +Property list +^^^^^^^^^^^^^ + +Gomill knows the types of all general and Go-specific |sgf| properties defined +in FF[4]: + +====== ========================== =================== + Id |sgf| type Meaning +====== ========================== =================== +``AB`` list of Stone Add Black +``AE`` list of Point Add Empty +``AN`` SimpleText Annotation +``AP`` SimpleText:SimpleText Application +``AR`` list of Point:Point Arrow +``AW`` list of Stone Add White +``B`` Move Black move +``BL`` Real Black time left +``BM`` Double Bad move +``BR`` SimpleText Black rank +``BT`` SimpleText Black team +``C`` Text Comment +``CA`` SimpleText Charset +``CP`` SimpleText Copyright +``CR`` list of Point Circle +``DD`` elist of Point Dim Points +``DM`` Double Even position +``DO`` None Doubtful +``DT`` SimpleText Date +``EV`` SimpleText Event +``FF`` Number File format +``FG`` None | Number:SimpleText Figure +``GB`` Double Good for Black +``GC`` Text Game comment +``GM`` Number Game +``GN`` SimpleText Game name +``GW`` Double Good for White +``HA`` Number Handicap +``HO`` Double Hotspot +``IT`` None Interesting +``KM`` Real Komi +``KO`` None Ko +``LB`` list of Point:SimpleText Label +``LN`` list of Point:Point Line +``MA`` list of Point Mark +``MN`` Number Set move number +``N`` SimpleText Node name +``OB`` Number Overtime stones left for Black +``ON`` SimpleText Opening +``OT`` SimpleText Overtime description +``OW`` Number Overtime stones left for White +``PB`` SimpleText Black player name +``PC`` SimpleText Place +``PL`` Colour Player to play +``PM`` Number Print move mode +``PW`` SimpleText White player name +``RE`` SimpleText Result +``RO`` SimpleText Round +``RU`` SimpleText Rules +``SL`` list of Point Selected +``SO`` SimpleText Source +``SQ`` list of Point Square +``ST`` Number Style +``SZ`` Number Size +``TB`` elist of Point Black territory +``TE`` Double Tesuji +``TM`` Real Time limit +``TR`` list of Point Triangle +``TW`` elist of Point White territory +``UC`` Double Unclear position +``US`` SimpleText User +``V`` Real Value +``VW`` elist of Point View +``W`` Move White move +``WL`` Real White time left +``WR`` SimpleText White rank +``WT`` SimpleText White team +====== ========================== =================== + + +.. _raw_property_encoding: + +Character encoding handling +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The |sgf| format is defined as containing ASCII-encoded data, possibly with +non-ASCII characters in Text and SimpleText property values. The Gomill +functions for loading and serialising |sgf| data work with 8-bit Python +strings. + +The encoding used for Text and SimpleText property values is given by the +``CA`` root property (if that isn't present, the encoding is ``ISO-8859-1``). + +In order for an encoding to be used in Gomill, it must exist as a Python +built-in codec, and it must be compatible with ASCII (at least whitespace, +``\``, ``]``, and ``:`` must be in the usual places). Behaviour is unspecified +if a non-ASCII-compatible encoding is requested. + +When encodings are passed as parameters (or returned from functions), they are +represented using the names or aliases of Python built-in codecs (eg +``"UTF-8"`` or ``"ISO-8859-1"``). See `standard encodings`__ for a list. +Values of the ``CA`` property are interpreted in the same way. + + .. __: http://docs.python.org/release/2.7/library/codecs.html#standard-encodings + +Each :class:`.Sgf_game` and :class:`.Tree_node` has a fixed :dfn:`raw property +encoding`, which is the encoding used internally to store the property values. +The :meth:`Tree_node.get_raw` and :meth:`Tree_node.set_raw` methods use the +raw property encoding. + +When an |sgf| game is loaded from a file, the raw property encoding is the +original file encoding (unless overridden). Improperly encoded property values +will not be detected until they are accessed (:meth:`~Tree_node.get` will +raise :exc:`ValueError`; use :meth:`~Tree_node.get_raw` to retrieve the actual +bytes). + + +.. _transcoding: + +.. rubric:: Transcoding + +When an |sgf| game is serialised to a string, the encoding represented by the +``CA`` root property is used. This :dfn:`target encoding` will be the same as +the raw property encoding unless ``CA`` has been changed since the +:class:`.Sgf_game` was created. + +When the raw property encoding and the target encoding match, the raw property +values are included unchanged in the output (even if they are improperly +encoded.) + +Otherwise, if any raw property value is improperly encoded, +:exc:`UnicodeDecodeError` is raised, and if any property value can't be +represented in the target encoding, :exc:`UnicodeEncodeError` is raised. + +If the target encoding doesn't identify a Python codec, :exc:`ValueError` is +raised. The behaviour of :meth:`~Sgf_game.serialise` is unspecified if the +target encoding isn't ASCII-compatible (eg, UTF-16). + + +.. _parsing_details: + +Parsing +^^^^^^^ + +The parser permits non-|sgf| content to appear before the beginning and after +the end of the game. It identifies the start of |sgf| content by looking for +``(;`` (with possible whitespace between the two characters). + +The parser accepts at most 8 letters in *PropIdents* (there is no formal limit +in the specification, but no standard property has more than 2). + +The parser doesn't perform any checks on property values. In particular, it +allows multiple values to be present for any property. + +The parser doesn't, in general, attempt to 'fix' ill-formed |sgf| content. As +an exception, if a *PropIdent* appears more than once in a node it is +converted to a single property with multiple values. + +The parser doesn't permit lower-case letters in *PropIdents* (these are +allowed in some ancient |sgf| variants). + + +The :mod:`!sgf_moves` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: gomill.sgf_moves + :synopsis: Higher-level processing of moves and positions from SGF games. + +The :mod:`!gomill.sgf_moves` module contains some higher-level functions for +processing moves and positions, and provides a link to the :mod:`.boards` +module. + + +.. function:: get_setup_and_moves(sgf_game[, board]) + + :rtype: tuple (:class:`.Board`, list of tuples (*colour*, *move*)) + + Returns the initial setup and the following moves from an + :class:`.Sgf_game`. + + The board represents the position described by ``AB`` and/or ``AW`` + properties in the |sgf| game's root node. :exc:`ValueError` is raised if + this position isn't legal. + + The moves are from the game's leftmost variation. Doesn't check that the + moves are legal. + + Raises :exc:`ValueError` if the game has structure it doesn't support. + + Currently doesn't support ``AB``/``AW``/``AE`` properties after the root + node. + + If the optional *board* parameter is provided, it must be an empty + :class:`.Board` of the right size; the same object will be returned (this + option is provided so you can use a different Board class). + + See also the :script:`show_sgf.py` example script. + + +.. function:: set_initial_position(sgf_game, board) + + Adds ``AB``/``AW``/``AE`` properties to an :class:`.Sgf_game`'s root node, + to reflect the position from a :class:`.Board`. + + Replaces any existing ``AB``/``AW``/``AE`` properties in the root node. + + +.. function:: indicate_first_player(sgf_game) + + Adds a ``PL`` property to an :class:`.Sgf_game`'s root node if appropriate, + to indicate which colour is first to play. + + Looks at the first child of the root to see who the first player is, and + sets ``PL`` it isn't the expected player (Black normally, but White if + there is a handicap), or if there are non-handicap setup stones. + diff --git a/gomill/docs/tournament_results.rst b/gomill/docs/tournament_results.rst new file mode 100644 index 0000000..0cbb49d --- /dev/null +++ b/gomill/docs/tournament_results.rst @@ -0,0 +1,401 @@ +Tournament results API +---------------------- + +.. module:: gomill.tournament_results + :synopsis: Retrieving and reporting on tournament results. + +This is a Python interface for processing the game results stored in a +tournament's :ref:`state file `. It can be used to write +custom reports, or to find games with particular results. + +Note that it can be used only for :ref:`tournaments ` (not for +:ref:`tuning events `). + +.. contents:: Page contents + :local: + :backlinks: none + + +General +^^^^^^^ + +In this interface, players are identified using their player codes (that is, +their keys in the control file's :setting:`players` dictionary). + +.. note:: In a :doc:`playoff tournament `, it is possible + to define a matchup in which the same player takes both colours. In this + case, the player code used for the second player will be the player code + from the control file with ``'#2'`` appended. + +The classes described here are implemented in the +:mod:`!gomill.tournament_results` and :mod:`!gomill.gtp_games` modules, but +you should not normally import these directly. See +:ref:`using_the_api_in_scripts`. + + +Tournament_results objects +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Tournament_results + + A Tournament_results object provides access to the game results and + statistics for a single tournament. + + The tournament results are catalogued in terms of :dfn:`matchups`, with + each matchup corresponding to a series of games which had the same players + and settings. Each matchup has an id, which is a short string. + + Tournament_results objects are normally retrieved from :class:`!Competition` + or :class:`!Ringmaster` objects; see :ref:`using_the_api_in_scripts`. + + Tournament_results objects support the following methods: + + .. method:: get_matchup_ids() + + :rtype: list of strings + + Return the tournament's matchup ids. + + .. method:: get_matchup(matchup_id) + + :rtype: :class:`Matchup_description` + + Describe the matchup with the specified id. + + .. method:: get_matchups() + + :rtype: map *matchup id* → :class:`Matchup_description` + + Describe all matchups. + + .. method:: get_matchup_stats(matchup_id) + + :rtype: :class:`Matchup_stats` object + + Return statistics for the matchup with the specified id. + + .. method:: get_matchup_results(matchup_id) + + :rtype: list of :class:`~.Game_result` objects + + Return the individual game results for the matchup with the specified id. + + The list is in unspecified order (in particular, the colours don't + necessarily alternate, even if :attr:`~Matchup_description.alternating` + is ``True`` for the matchup). + + :ref:`void games` do not appear in these results. + + +Matchup_description objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Matchup_description + + A Matchup_description describes a series of games which had the same + players and settings. The information comes from the current contents of + the tournament's control file. + + Matchup_descriptions are normally retrieved from + :class:`Tournament_results` objects. + + Matchup_descriptions have the following attributes (which should be treated + as read-only): + + .. attribute:: id + + The :ref:`matchup id ` (a string, usually 1 to 3 characters). + + .. attribute:: player_1 + player_2 + + The :ref:`player codes ` of the two players. These are + never equal. + + .. attribute:: name + + String describing the matchup (eg ``'xxx v yyy'``). + + .. attribute:: board_size + + Integer (eg ``19``). + + .. attribute:: komi + + Float (eg ``7.0``). + + .. attribute:: alternating + + Bool. If this is ``False``, :attr:`player_1` played black and + :attr:`player_2` played white; otherwise they alternated. + + .. attribute:: handicap + + Integer or ``None``. + + .. attribute:: handicap_style + + String: ``'fixed'`` or ``'free'``. + + .. attribute:: move_limit + + Integer or ``None``. See :ref:`playing games`. + + .. attribute:: scorer + + String: ``'internal'`` or ``'players'``. See :ref:`scoring`. + + .. attribute:: number_of_games + + Integer or ``None``. This is the number of games requested in the + control file; it may not match the number of game results that are + available. + + + Matchup_descriptions support the following method: + + .. method:: describe_details() + + :rtype: string + + Return a text description of the matchup's game settings. + + This covers the most important game settings which can't be observed in + the results table (board size, handicap, and komi). + + +Matchup_stats objects +^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Matchup_stats + + A Matchup_stats object provides basic summary information for a matchup. + The information comes from the tournament's :ref:`state file `. + + Matchup_stats objects are normally retrieved from + :class:`Tournament_results` objects. + + Matchup_stats objects have the following attributes (which should be + treated as read-only): + + .. attribute:: player_1 + player_2 + + The :ref:`player codes ` of the two players. These are + never equal. + + .. attribute:: total + + Integer. The number of games played in the matchup. + + .. attribute:: wins_1 + wins_2 + + Integer. The number of games won by each player. + + .. attribute:: forfeits_1 + forfeits_2 + + Integer. The number of games in which each player lost by forfeit. + + .. attribute:: unknown + + Integer. The number of games whose result is unknown. + + .. attribute:: average_time_1 + average_time_2 + + float or ``None``. The average CPU time taken by each player. + + If CPU times are available for only some games, the average is taken + over the games for which they are available. If they aren't available + for any games, the average is given as ``None``. See :ref:`cpu time` + for notes on how CPU times are obtained. + + .. attribute:: played_1b + played_2b + + Integer. The number of games in which each player took Black. + + .. attribute:: played_1w + played_2w + + Integer. The number of games in which each player took White. + + .. attribute:: alternating + + Bool. This is true if each player played at least one game as Black and + at least one game as White. + + This doesn't always equal the :attr:`~Matchup_description.alternating` + attribute from the corresponding :class:`Matchup_description` object (in + particular, if only one game was played in the matchup, it will always + be ``False``). + + If :attr:`alternating` is ``True``, the following attributes are also + available: + + .. attribute:: wins_b + + Integer. The number of games in which Black won. + + .. attribute:: wins_w + + Integer. The number of games in which White won. + + .. attribute:: wins_1b + wins_2b + + Integer. The number of games in which each player won with Black. + + .. attribute:: wins_1w + wins_2w + + Integer. The number of games in which each player won with White. + + + If :attr:`alternating` is ``False``, the following attributes are also + available: + + .. attribute:: colour_1 + colour_2 + + The *colour* taken by each player. + + +.. currentmodule:: gomill.gtp_games + +Game_result objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Game_result + + A Game_result contains the information recorded for an individual game. The + information comes from the tournament's :ref:`state file `. + + .. note:: If an |sgf| :ref:`game record ` has been written + for the game, you can retrieve its location in the filesystem from a + :class:`!Ringmaster` object using + :samp:`ringmaster.get_sgf_pathname({game_id})`. + + The :ref:`player codes ` used here are the same as the ones + in the corresponding :class:`.Matchup_description`'s + :attr:`~.Matchup_description.player_1` and + :attr:`~.Matchup_description.player_2` attributes. + + See :ref:`playing games` and :ref:`details of scoring` for an explanation + of the possible game results. Games with unknown result can be + distinguished as having :attr:`winning_player` ``None`` but :attr:`is_jigo` + ``False``. + + Game_results can be retrieved from + :class:`.Tournament_results` objects. + + Game_results have the following attributes (which should be treated as + read-only): + + .. attribute:: game_id + + Short string uniquely identifying the game within the tournament. See + :ref:`game id`. + + .. Game_results returned via Tournament_results always have game_id set, + so documenting it that way here. + + .. attribute:: players + + Map *colour* → :ref:`player code `. + + .. attribute:: player_b + + :ref:`player code ` of the Black player. + + .. attribute:: player_w + + :ref:`player code ` of the White player. + + .. attribute:: winning_player + + :ref:`player code ` or ``None``. + + .. attribute:: losing_player + + :ref:`player code ` or ``None``. + + .. attribute:: winning_colour + + *colour* or ``None``. + + .. attribute:: losing_colour + + *colour* or ``None``. + + .. attribute:: is_jigo + + Bool: ``True`` if the game was a :term:`jigo`. + + .. attribute:: is_forfeit + + Bool: ``True`` if one of the players lost the game by forfeit; see + :ref:`playing games`. + + .. attribute:: sgf_result + + String describing the game's result. This is in the format used for the + :term:`SGF` ``RE`` property (eg ``'B+1.5'``). + + .. attribute:: detail + + Additional information about the game result (string or ``None``). + + This is present (not ``None``) for those game results which are not wins + on points, jigos, or wins by resignation. + + .. (leaving cpu_times undocumented, as I don't want to say it's stable) + + .. attribute:: cpu_times + + Map :ref:`player code ` → *time*. + + The time is a float representing a number of seconds, or ``None`` if + time is not available, or ``'?'`` if :gtp:`gomill-cpu_time` is + implemented but returned a failure response. + + See :ref:`cpu time` for more details. + + + Game_results support the following method: + + .. method:: describe() + + :rtype: string + + Return a short human-readable description of the result. + + For example, ``'xxx beat yyy (W+2.5)'``. + + +.. currentmodule:: tournament_results + +.. _using_the_api_in_scripts: + +Using the API in scripts +^^^^^^^^^^^^^^^^^^^^^^^^ + +To write a standalone script using the tournaments results API, obtain a +:class:`.Tournament_results` object from a :class:`!Ringmaster` object as +follows:: + + from gomill import ringmasters + ringmaster = ringmasters.Ringmaster(control_file_pathname) + ringmaster.load_status() + tournament_results = ringmaster.tournament_results() + +All of these calls report problems by raising the :exc:`!RingmasterError` +exception defined in the :mod:`!ringmasters` module. + +See the :script:`find_forfeits.py` example script for a more fleshed-out +example. + diff --git a/gomill/examples/clop_example.ctl b/gomill/examples/clop_example.ctl new file mode 100644 index 0000000..e19d7bb --- /dev/null +++ b/gomill/examples/clop_example.ctl @@ -0,0 +1,78 @@ +# The following settings are supported: +# +# - all _common settings_ +# +# - all _game settings_ +# +# - tuning event settings (cf mcts_tuner): +# - candidate_colour +# - opponent +# - parameters +# - make_candidate +# +# - settings for experiment control +# - parallel -- number of games to run in parallel +# - stop_on_error -- boolean +# +# - regression parameters: +# - clop_H -- float +# - correlations -- 'all' (default) or 'none' +# +## << +# clop_H: 3 is recommended (it is the default value) +# correlations: +# Even if variables are not correlated "all" should work well. The problem is +# that the regression might become very costly if the number of variables is +# high. So use "correlations none" only if you are certain parameters are +# independent or you have so many variables that "all" is too costly. +## >> +# + +# The available parameter types are: +# LinearParameter +# IntegerParameter +# GammaParameter +# IntegerGammaParameter +# For GammaParameter, quadratic regression is performed on log(x) + +competition_type = "clop_tuner" + +description = """\ +Sample control file for CLOP integration. + +""" + +def gnugo(level): + return Player("gnugo --mode=gtp --chinese-rules --capture-all-dead " + "--level=%d" % level) + +def pachi(playouts, policy): + return Player( + "~/src/pachi/pachi " + "-d 0 " # silence stderr + "-t =%d " + "threads=1,max_tree_size=2048 " + "policy=%s " + % (playouts, policy)) + +players = { + 'gnugo-l7' : gnugo(7), + } + + +parameters = [ + Parameter('equiv_rave', + type = "GammaParameter", + min = 40, + max = 32000), + ] + +def make_candidate(equiv_rave): + return pachi(2000, policy="ucb1amaf:equiv_rave=%f" % equiv_rave) + +board_size = 19 +komi = 7.5 +opponent = 'gnugo-l7' +candidate_colour = 'w' +parallel = 2 + diff --git a/gomill/examples/find_forfeits.py b/gomill/examples/find_forfeits.py new file mode 100644 index 0000000..a909b73 --- /dev/null +++ b/gomill/examples/find_forfeits.py @@ -0,0 +1,54 @@ +"""Find forfeited games in tournament results. + +This demonstrates retrieving and processing results from a tournament. + +""" + +import sys +from optparse import OptionParser + +from gomill.common import opponent_of +from gomill.ringmasters import Ringmaster, RingmasterError + +def show_result(matchup, result, filename): + print "%s: %s forfeited game %s" % ( + matchup.name, result.losing_player, filename) + +def find_forfeits(ringmaster): + ringmaster.load_status() + tournament_results = ringmaster.get_tournament_results() + matchup_ids = tournament_results.get_matchup_ids() + for matchup_id in matchup_ids: + matchup = tournament_results.get_matchup(matchup_id) + results = tournament_results.get_matchup_results(matchup_id) + for result in results: + if result.is_forfeit: + filename = ringmaster.get_sgf_filename(result.game_id) + show_result(matchup, result, filename) + + +_description = """\ +Read results of a tournament and show all forfeited games. +""" + +def main(argv): + parser = OptionParser(usage="%prog ", + description=_description) + opts, args = parser.parse_args(argv) + if not args: + parser.error("not enough arguments") + if len(args) > 1: + parser.error("too many arguments") + ctl_pathname = args[0] + try: + ringmaster = Ringmaster(ctl_pathname) + find_forfeits(ringmaster) + except RingmasterError, e: + print >>sys.stderr, "ringmaster:" + print >>sys.stderr, e + sys.exit(1) + + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/gomill/examples/gomill-clop b/gomill/examples/gomill-clop new file mode 100755 index 0000000..a2716d8 --- /dev/null +++ b/gomill/examples/gomill-clop @@ -0,0 +1,470 @@ +#!/usr/bin/env python +"""Gomill integration for CLOP. + +Designed for use with CLOP 0.0.8, available from +http://remi.coulom.free.fr/CLOP/ + +""" + +# The 'connection script' interface is as follows: +# - the command-line arguments are: +# - any arguments specified in the Script line +# - processor +# - seed +# - then pairs of arguments (parameter name, parameter value) +# - the connection script prints a single character to stdout: +# 'W' for candidate win, 'D' for draw, 'L' for loss +# any further output is ignored +# - if the output doesn't start with 'W', 'D', or 'L', it's treated as an error, +# and the complete stdout and stderr are reported +# (by convention, print "Error:" with a description) +# - the connection script's exit status is ignored + +import os +import sys +from optparse import OptionParser + +from gomill import compact_tracebacks +from gomill import competitions +from gomill import game_jobs +from gomill.competitions import ( + Competition, CompetitionError, ControlFileError, Player_config) +from gomill.job_manager import JobFailed +from gomill.ringmasters import ( + Ringmaster, RingmasterError, RingmasterInternalError) +from gomill.settings import * +from gomill.common import opponent_of + + +PARAMETER_TYPES = [ + "LinearParameter", + "IntegerParameter", + "GammaParameter", + "IntegerGammaParameter", + ] + +parameter_settings = [ + Setting('code', interpret_identifier), + Setting('type', interpret_enum(*PARAMETER_TYPES)), + Setting('min', interpret_float), + Setting('max', interpret_float), + ] + +class Parameter_config(Quiet_config): + """Parameter (ie, dimension) description for use in control files.""" + # positional or keyword + positional_arguments = ('code',) + # keyword-only + keyword_arguments = tuple(setting.name for setting in parameter_settings + if setting.name != 'code') + +class Parameter_spec(object): + """Internal description of a parameter spec from the configuration file. + + Public attributes: + code -- identifier + + """ + def format_for_clop(self): + """Return the parameter configuration string for the .clop file.""" + if self.is_integer: + fmt = "%s %s %d %d" + else: + fmt = "%s %s %f %f" + return fmt % (self.type, self.code, self.min, self.max) + + def interpret_value(self, s): + """Convert CLOP command-line parameter to an engine parameter. + + Returns an int or float. + + """ + if self.is_integer: + return int(s) + else: + return float(s) + + def format_for_display(self, v): + return str(v) + +class Clop_tuner(Competition): + + def control_file_globals(self): + result = Competition.control_file_globals(self) + result.update({ + 'Parameter' : Parameter_config, + }) + return result + + global_settings = (Competition.global_settings + + competitions.game_settings + [ + Setting('candidate_colour', interpret_colour), + Setting('parallel', interpret_int, default=1), + Setting('clop_H', interpret_float, default=3), + Setting('correlations', interpret_enum('all', 'none'), default='all'), + Setting('stop_on_error', interpret_bool, default=True), + ]) + + special_settings = [ + Setting('opponent', interpret_identifier), + Setting('parameters', + interpret_sequence_of_quiet_configs(Parameter_config)), + Setting('make_candidate', interpret_callable), + ] + + def parameter_spec_from_config(self, parameter_config): + """Make a Parameter_spec from a Parameter_config. + + Raises ControlFileError if there is an error in the configuration. + + Returns a Parameter_spec with all attributes set. + + """ + arguments = parameter_config.resolve_arguments() + interpreted = load_settings(parameter_settings, arguments) + pspec = Parameter_spec() + for name, value in interpreted.iteritems(): + setattr(pspec, name, value) + pspec.is_integer = ("Integer" in pspec.type) + if pspec.is_integer: + if pspec.min != int(pspec.min): + raise ControlFileError("'min': should be an integer") + if pspec.max != int(pspec.max): + raise ControlFileError("'max': should be an integer") + return pspec + + def initialise_from_control_file(self, config): + Competition.initialise_from_control_file(self, config) + + competitions.validate_handicap( + self.handicap, self.handicap_style, self.board_size) + + try: + specials = load_settings(self.special_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + + try: + self.opponent = self.players[specials['opponent']] + except KeyError: + raise ControlFileError( + "opponent: unknown player %s" % specials['opponent']) + + self.parameter_specs = [] + if not specials['parameters']: + raise ControlFileError("parameters: empty list") + seen_codes = set() + for i, parameter_spec in enumerate(specials['parameters']): + try: + pspec = self.parameter_spec_from_config(parameter_spec) + except StandardError, e: + code = parameter_spec.get_key() + if code is None: + code = i + raise ControlFileError("parameter %s: %s" % (code, e)) + if pspec.code in seen_codes: + raise ControlFileError( + "duplicate parameter code: %s" % pspec.code) + seen_codes.add(pspec.code) + self.parameter_specs.append(pspec) + self.candidate_maker_fn = specials['make_candidate'] + + def get_clop_parameter_specs(self): + """Describe the parameters in the format used in the .clop file. + + Returns a list of strings. + + """ + return [pspec.format_for_clop() for pspec in self.parameter_specs] + + def interpret_clop_parameters(self, clop_parameters): + """Convert the CLOP command-line parameters to engine parameters. + + clop_parameters -- list of pairs of strings + (parameter name, parameter value) + + Returns a list of engine parameters, suitable for passing to + make_candidate(). + + """ + engine_parameters = [] + try: + if len(clop_parameters) != len(self.parameter_specs): + raise ValueError + for pspec, (name, value) in \ + zip(self.parameter_specs, clop_parameters): + if name != pspec.code: + raise ValueError + engine_parameters.append(pspec.interpret_value(value)) + return engine_parameters + except ValueError: + raise CompetitionError( + "couldn't interpret parameters: %s" % repr(clop_parameters)) + + def format_engine_parameters(self, engine_parameters): + return "; ".join( + "%s %s" % (pspec.code, pspec.format_for_display(v)) + for pspec, v in zip(self.parameter_specs, engine_parameters)) + + def make_candidate(self, player_code, engine_parameters): + """Make a player using the specified engine parameters. + + Returns a game_jobs.Player. + + """ + try: + candidate_config = self.candidate_maker_fn(*engine_parameters) + except Exception: + raise CompetitionError( + "error from make_candidate()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + if not isinstance(candidate_config, Player_config): + raise CompetitionError( + "make_candidate() returned %r, not Player" % + candidate_config) + try: + candidate = self.game_jobs_player_from_config( + player_code, candidate_config) + except Exception, e: + raise CompetitionError( + "bad player spec from make_candidate():\n" + "%s\nparameters were: %s" % + (e, self.format_engine_parameters(engine_parameters))) + return candidate + + def get_game_for_parameters(self, clop_seed, clop_parameters): + """Return the details of the next game to play. + + clop_seed -- second command-line parameter passed by clop + clop_parameters -- remaining command-line parameters passed by clop. + + This is like Competition.get_game(), but it never returns + NoGameAvailable. + + """ + engine_parameters = self.interpret_clop_parameters(clop_parameters) + + candidate = self.make_candidate("#%s" % clop_seed, engine_parameters) + job = game_jobs.Game_job() + job.game_id = clop_seed + if self.candidate_colour == 'b': + job.player_b = candidate + job.player_w = self.opponent + else: + job.player_b = self.opponent + job.player_w = candidate + job.board_size = self.board_size + job.komi = self.komi + job.move_limit = self.move_limit + job.handicap = self.handicap + job.handicap_is_free = (self.handicap_style == 'free') + job.use_internal_scorer = (self.scorer == 'internal') + job.internal_scorer_handicap_compensation = \ + self.internal_scorer_handicap_compensation + job.sgf_event = self.competition_code + job.sgf_note = ("Candidate parameters: %s" % + self.format_engine_parameters(engine_parameters)) + return job + + + +class Clop_ringmaster(Ringmaster): + def __init__(self, *args, **kwargs): + Ringmaster.__init__(self, *args, **kwargs) + # clop uses .log, so we need something different + self.log_pathname = os.path.join( + self.base_directory, self.competition_code) + ".elog" + + @staticmethod + def _get_competition_class(competition_type): + if competition_type == "clop_tuner": + return Clop_tuner + else: + raise ValueError + + def ensure_output_directories(self): + if self.record_games: + try: + if not os.path.exists(self.sgf_dir_pathname): + os.mkdir(self.sgf_dir_pathname) + except EnvironmentError: + raise RingmasterError("failed to create SGF directory:\n%s" % e) + + if self.write_gtp_logs: + try: + if not os.path.exists(self.gtplog_dir_pathname): + os.mkdir(self.gtplog_dir_pathname) + except EnvironmentError: + raise RingmasterError( + "failed to create GTP log directory:\n%s" % e) + + def open_logfile(self): + try: + self.logfile = open(self.log_pathname, "a") + except EnvironmentError, e: + raise RingmasterError("failed to open log file:\n%s" % e) + + def run_game_for_clop(self, seed, parameters): + """Act as a CLOP connection script. + + seed -- seed string passed by clop + parameters -- list of pairs of strings (parameter name, parameter value) + + Returns the message to print. + + """ + self._initialise_presenter() + try: + job = self.competition.get_game_for_parameters(seed, parameters) + self._prepare_job(job) + start_msg = "starting game %s: %s (b) vs %s (w)" % ( + job.game_id, job.player_b.code, job.player_w.code) + self.log(start_msg) + response = job.run() + self.log("response from game %s" % response.game_id) + for warning in response.warnings: + self.warn(warning) + for log_entry in response.log_entries: + self.log(log_entry) + except (CompetitionError, JobFailed), e: + raise RingmasterError(e) + result = response.game_result + candidate_colour = self.competition.candidate_colour + if result.winning_colour == candidate_colour: + message = "W" + elif result.winning_colour == opponent_of(candidate_colour): + message = "L" + elif result.is_jigo: + message = "D" + else: + if self.competition.stop_on_error: + # Don't want the experiment to stop just because a single game + # failed (eg, went over the move limit), so treat it as a draw. + message = "D" + else: + raise RingmasterError("unexpected game result: %s" % + result.sgf_result) + return message + + + +clop_template = """\ +Name %(experiment_name)s + +Script %(connection_pathname)s %(control_filename)s run-game + +%(parameter_specs)s + +%(processor_specs)s + +Replications 1 + +DrawElo %(drawelo)s + +H %(clop_H)s + +Correlations %(correlations)s + +StopOnError %(stop_on_error)s + +""" + +def do_setup(ringmaster, arguments, options): + """Create the .clop file, and any needed directories.""" + connection_pathname = os.path.abspath(__file__) + control_filename = os.path.basename(ringmaster.control_pathname) + clop_pathname = os.path.join(ringmaster.base_directory, + "%s.clop" % ringmaster.competition_code) + competition = ringmaster.competition + experiment_name = ringmaster.competition_code + + parameter_specs = "\n".join(competition.get_clop_parameter_specs()) + processor_specs = "\n".join( + "Processor par%d" % i for i in xrange(competition.parallel)) + if competition.komi == int(competition.komi) or competition.stop_on_error: + drawelo = "100" + else: + drawelo = "0" + clop_H = competition.clop_H + correlations = competition.correlations + stop_on_error = "true" if competition.stop_on_error else "false" + + with open(clop_pathname, "w") as f: + f.write(clop_template % locals()) + + ringmaster.ensure_output_directories() + +def do_run_game(ringmaster, arguments, options): + """Act as a CLOP connection script.""" + try: + processor, seed = arguments[:2] + except ValueError: + raise RingmasterError("not enough connection script arguments") + parameter_args = arguments[2:] + parameters = [] + i = 0 + try: + while i < len(parameter_args): + parameters.append((parameter_args[i], parameter_args[i+1])) + i += 2 + except LookupError: + raise RingmasterError("parameter without value: %s" % parameter_args[i]) + ringmaster.set_display_mode('quiet') + ringmaster.open_logfile() + message = ringmaster.run_game_for_clop(seed, parameters) + print message + +_actions = { + "setup" : (do_setup, False), + "run-game" : (do_run_game, True), + } + +def main(argv): + usage = ("%prog [connection script arguments]\n\n" + "commands: setup, run-game") + description = ( + "`setup` generates a .clop file for use with clop-gui or clop-console. " + "Then `run-game` is used (behind the scenes) as the connection script.") + parser = OptionParser(usage=usage, description=description) + (options, args) = parser.parse_args(argv) + if len(args) == 0: + parser.error("no control file specified") + if len(args) == 1: + parser.error("no command specified") + command = args[1] + command_args = args[2:] + try: + action, takes_arguments = _actions[command] + except KeyError: + parser.error("no such command: %s" % command) + if command_args and not takes_arguments: + parser.error("too many arguments for %s" % command) + + ctl_pathname = args[0] + try: + if not os.path.exists(ctl_pathname): + raise RingmasterError("control file %s not found" % ctl_pathname) + ringmaster = Clop_ringmaster(ctl_pathname) + action(ringmaster, command_args, options) + exit_status = 0 + except RingmasterError, e: + print >>sys.stderr, "gomill-clop:", e + exit_status = 1 + except KeyboardInterrupt: + exit_status = 3 + except RingmasterInternalError, e: + print >>sys.stderr, "gomill-clop: internal error" + print >>sys.stderr, e + exit_status = 4 + except: + print >>sys.stderr, "gomill-clop: internal error" + compact_tracebacks.log_traceback() + exit_status = 4 + if exit_status != 0 and command == 'run-game': + # Make sure the problem runner sees an error response + print "Error" + sys.exit(exit_status) + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/gomill/examples/gtp_stateful_player b/gomill/examples/gtp_stateful_player new file mode 100755 index 0000000..ed85c61 --- /dev/null +++ b/gomill/examples/gtp_stateful_player @@ -0,0 +1,118 @@ +#!/usr/bin/env python +"""GTP engine which maintains the board position. + +This provides an example of a GTP engine using the gtp_states module. + +It plays (and resigns) randomly. + +It supports the following GTP commands, mostly provided by gtp_states: + +Standard + boardsize + clear_board + fixed_handicap + genmove + known_command + komi + list_commands + loadsgf + name + place_free_handicap + play + protocol_version + quit + reg_genmove + set_free_handicap + showboard + undo + version + +Gomill extensions + gomill-explain_last_move + gomill-genmove_ex + gomill-savesgf + +Examples + gomill_resign_p -- resign in future with the specified probabiltiy + +""" + +import random +import sys + +from gomill import gtp_engine +from gomill import gtp_states + + + +class Player(object): + """Player for use with gtp_state.""" + + def __init__(self): + self.resign_probability = 0.1 + + def genmove(self, game_state, player): + """Move generator that chooses a random empty point. + + game_state -- gtp_states.Game_state + player -- 'b' or 'w' + + This may return a self-capture move. + + """ + board = game_state.board + empties = [] + for row, col in board.board_points: + if board.get(row, col) is None: + empties.append((row, col)) + result = gtp_states.Move_generator_result() + if random.random() < self.resign_probability: + result.resign = True + else: + result.move = random.choice(empties) + # Used by gomill-explain_last_move and gomill-savesgf + result.comments = "chosen at random from %d choices" % len(empties) + return result + + def handle_name(self, args): + return "GTP stateful player" + + def handle_version(self, args): + return "" + + def handle_resign_p(self, args): + try: + f = gtp_engine.interpret_float(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + self.resign_probability = f + + def get_handlers(self): + return { + 'name' : self.handle_name, + 'version' : self.handle_version, + 'gomill-resign_p' : self.handle_resign_p, + } + + +def make_engine(player): + """Return a Gtp_engine_protocol which runs the specified player.""" + gtp_state = gtp_states.Gtp_state( + move_generator=player.genmove, + acceptable_sizes=(9, 13, 19)) + engine = gtp_engine.Gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_commands(gtp_state.get_handlers()) + engine.add_commands(player.get_handlers()) + return engine + +def main(): + try: + player = Player() + engine = make_engine(player) + gtp_engine.run_interactive_gtp_session(engine) + except (KeyboardInterrupt, gtp_engine.ControllerDisconnected): + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/gomill/examples/gtp_test_player b/gomill/examples/gtp_test_player new file mode 100755 index 0000000..fb801cf --- /dev/null +++ b/gomill/examples/gtp_test_player @@ -0,0 +1,144 @@ +#!/usr/bin/env python +"""GTP engine intended for testing GTP controllers. + +This provides an example of a GTP engine which does not use the gtp_states +module. + +It supports the following GTP commands: + +Standard + boardsize + clear_board + genmove + known_command + komi + list_commands + name + play + protocol_version + quit + version + +Extensions: + gomill-force_error [error_type] + gomill-delayed_error [error_type] + +gomill-force_error immediately causes an error. error_type can be any of the +following: + error -- return a GTP error response (this is the default) + exit -- return a GTP error response and end the GTP session + internal -- propagate a Python exception to the GTP engine code + kill -- abruptly terminate the engine process + protocol -- send an ill-formed GTP response + +gomill-delayed_error causes a later genmove command to produce an error. This +will happen the first time genmove is called for the move 'move_number' or +later, counting from the start of the game. + +""" + +import os +import sys + +from gomill import gtp_engine +from gomill.gtp_engine import GtpError, GtpFatalError + + +class Test_player(object): + """GTP player used for testing controllers' error handling.""" + def __init__(self): + self.delayed_error_move = None + self.delayed_error_args = None + self.move_count = 0 + + def handle_name(self, args): + return "GTP test player" + + def handle_version(self, args): + return "" + + def handle_genmove(self, args): + """Handler for the genmove command. + + This honours gomill-delayed_error, and otherwise passes. + + """ + self.move_count += 1 + if (self.delayed_error_move and + self.move_count >= self.delayed_error_move): + self.delayed_error_move = None + self.handle_force_error(self.delayed_error_args) + return "pass" + + def handle_play(self, args): + self.move_count += 1 + + def handle_boardsize(self, args): + pass + + def handle_clear_board(self, args): + pass + + def handle_komi(self, args): + pass + + def handle_force_error(self, args): + """Handler for the gomill-force_error command.""" + try: + arg = args[0] + except IndexError: + arg = "error" + if arg == "error": + raise GtpError("forced GTP error") + if arg == "exit": + raise GtpFatalError("forced GTP error; exiting") + if arg == "internal": + 3 / 0 + if arg == "kill": + os.kill(os.getpid(), 15) + if arg == "protocol": + sys.stdout.write("!! forced ill-formed GTP response\n") + sys.stdout.flush() + return + raise GtpError("unknown force_error argument") + + def handle_delayed_error(self, args): + """Handler for the gomill-delayed_error command.""" + try: + move_number = gtp_engine.interpret_int(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + self.delayed_error_move = move_number + self.delayed_error_args = args[1:] + + def get_handlers(self): + return { + 'name' : self.handle_name, + 'version' : self.handle_version, + 'genmove' : self.handle_genmove, + 'play' : self.handle_play, + 'boardsize' : self.handle_boardsize, + 'clear_board' : self.handle_clear_board, + 'komi' : self.handle_komi, + 'gomill-force_error' : self.handle_force_error, + 'gomill-delayed_error' : self.handle_delayed_error, + } + + +def make_engine(test_player): + """Return a Gtp_engine_protocol which runs the specified Test_player.""" + engine = gtp_engine.Gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_commands(test_player.get_handlers()) + return engine + +def main(): + try: + test_player = Test_player() + engine = make_engine(test_player) + gtp_engine.run_interactive_gtp_session(engine) + except (KeyboardInterrupt, gtp_engine.ControllerDisconnected): + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/gomill/examples/kgs_proxy.py b/gomill/examples/kgs_proxy.py new file mode 100644 index 0000000..e93b12d --- /dev/null +++ b/gomill/examples/kgs_proxy.py @@ -0,0 +1,146 @@ +"""GTP proxy for use with kgsGtp. + +This supports saving a game record after each game, if the underlying engine +supports gomill-savesgf. + +""" +import os +import sys +from optparse import OptionParser + +from gomill import gtp_engine +from gomill import gtp_proxy +from gomill.gtp_engine import GtpError +from gomill.gtp_controller import BadGtpResponse + +class Kgs_proxy(object): + """GTP proxy for use with kgsGtp. + + Instantiate with command line arguments. + + Calls sys.exit on fatal errors. + + """ + + def __init__(self, command_line_args): + parser = OptionParser(usage="%prog [options] [args]") + parser.disable_interspersed_args() + parser.add_option("--sgf-dir", metavar="PATHNAME") + parser.add_option("--filename-template", metavar="TEMPLATE", + help="eg '%03d.sgf'") + opts, args = parser.parse_args(command_line_args) + + if not args: + parser.error("must specify a command") + self.subprocess_command = args + + self.filename_template = "%04d.sgf" + try: + opts.filename_template % 3 + except Exception: + pass + else: + self.filename_template = opts.filename_template + + self.sgf_dir = opts.sgf_dir + if self.sgf_dir: + self.check_sgf_dir() + self.do_savesgf = True + else: + self.do_savesgf = False + + def log(self, s): + print >>sys.stderr, s + + def run(self): + self.proxy = gtp_proxy.Gtp_proxy() + try: + self.proxy.set_back_end_subprocess(self.subprocess_command) + self.proxy.engine.add_commands( + {'genmove' : self.handle_genmove, + 'kgs-game_over' : self.handle_game_over, + }) + if (self.do_savesgf and + not self.proxy.back_end_has_command("gomill-savesgf")): + sys.exit("kgs_proxy: back end doesn't support gomill-savesgf") + + # Colour that we appear to be playing + self.my_colour = None + self.initialise_name() + except gtp_proxy.BackEndError, e: + sys.exit("kgs_proxy: %s" % e) + try: + self.proxy.run() + except KeyboardInterrupt: + sys.exit(1) + + def initialise_name(self): + def shorten_version(name, version): + """Clean up redundant version strings.""" + if version.lower().startswith(name.lower()): + version = version[len(name):].lstrip() + # For MoGo's stupidly long version string + a, b, c = version.partition(". Please read http:") + if b: + version = a + return version[:32].rstrip() + + self.my_name = None + try: + self.my_name = self.proxy.pass_command("name", []) + version = self.proxy.pass_command("version", []) + version = shorten_version(self.my_name, version) + self.my_name += ":" + version + except BadGtpResponse: + pass + + def handle_genmove(self, args): + try: + self.my_colour = gtp_engine.interpret_colour(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + return self.proxy.pass_command("genmove", args) + + def check_sgf_dir(self): + if not os.path.isdir(self.sgf_dir): + sys.exit("kgs_proxy: can't find save game directory %s" % + self.sgf_dir) + + def choose_filename(self, existing): + existing = set(existing) + for i in xrange(10000): + filename = self.filename_template % i + if filename not in existing: + return filename + raise StandardError("too many sgf files") + + def handle_game_over(self, args): + """Handler for kgs-game_over. + + kgsGtp doesn't send any arguments, so we don't know the result. + + """ + def escape_for_savesgf(s): + return s.replace("\\", "\\\\").replace(" ", "\\ ") + + if self.do_savesgf: + filename = self.choose_filename(os.listdir(self.sgf_dir)) + pathname = os.path.join(self.sgf_dir, filename) + self.log("kgs_proxy: saving game record to %s" % pathname) + args = [pathname] + if self.my_colour is not None and self.my_name is not None: + args.append("P%s=%s" % (self.my_colour.upper(), + escape_for_savesgf(self.my_name))) + try: + self.proxy.handle_command("gomill-savesgf", args) + except GtpError, e: + # Hide error from kgsGtp, though I don't suppose it would care + self.log("error: %s" % e) + + +def main(): + kgs_proxy = Kgs_proxy(sys.argv[1:]) + kgs_proxy.run() + +if __name__ == "__main__": + main() diff --git a/gomill/examples/mogo_wrapper.py b/gomill/examples/mogo_wrapper.py new file mode 100644 index 0000000..b2f8fe3 --- /dev/null +++ b/gomill/examples/mogo_wrapper.py @@ -0,0 +1,46 @@ +"""Proxy for making mogo a better-behaved GTP engine. + +This means the controller sees gomill's GTP implementation, not mogo's. + +This makes quarry willing to run mogo, for example. + +""" + +import sys + +from gomill import gtp_engine +from gomill import gtp_proxy + +def handle_version(args): + # Override remarkably verbose version response + return "2007 public release" + +def main(executable): + try: + if sys.argv[1] not in ("--9", "--13", "--19"): + raise ValueError + size = sys.argv[1][2:] + except Exception: + sys.exit("mogo_wrapper: first parameter must be --9, --13, or --19") + + def handle_boardsize(args): + # No need to pass this down to mogo. + try: + if args[0] != size: + raise gtp_engine.GtpError("board size %s only please" % size) + except IndexError: + gtp_engine.report_bad_arguments() + + proxy = gtp_proxy.Gtp_proxy() + proxy.set_back_end_subprocess([executable] + sys.argv[1:]) + proxy.engine.add_command("version", handle_version) + proxy.engine.add_command("boardsize", handle_boardsize) + proxy.pass_command("boardsize", [size]) + try: + proxy.run() + except KeyboardInterrupt: + sys.exit(1) + +if __name__ == "__main__": + main("mogo") + diff --git a/gomill/examples/show_sgf.py b/gomill/examples/show_sgf.py new file mode 100644 index 0000000..195d6d4 --- /dev/null +++ b/gomill/examples/show_sgf.py @@ -0,0 +1,72 @@ +"""Show the position from an SGF file. + +This demonstrates the sgf and ascii_boards modules. + +""" + +import sys +from optparse import OptionParser + +from gomill import ascii_boards +from gomill import sgf +from gomill import sgf_moves + +def show_sgf_file(pathname, move_number): + f = open(pathname) + sgf_src = f.read() + f.close() + try: + sgf_game = sgf.Sgf_game.from_string(sgf_src) + except ValueError: + raise StandardError("bad sgf file") + + try: + board, plays = sgf_moves.get_setup_and_moves(sgf_game) + except ValueError, e: + raise StandardError(str(e)) + if move_number is not None: + move_number = max(0, move_number-1) + plays = plays[:move_number] + + for colour, move in plays: + if move is None: + continue + row, col = move + try: + board.play(row, col, colour) + except ValueError: + raise StandardError("illegal move in sgf file") + + print ascii_boards.render_board(board) + print + +_description = """\ +Show the position from an SGF file. If a move number is specified, the position +before that move is shown (this is to match the behaviour of GTP loadsgf). +""" + +def main(argv): + parser = OptionParser(usage="%prog [move number]", + description=_description) + opts, args = parser.parse_args(argv) + if not args: + parser.error("not enough arguments") + pathname = args[0] + if len(args) > 2: + parser.error("too many arguments") + if len(args) == 2: + try: + move_number = int(args[1]) + except ValueError: + parser.error("invalid integer value: %s" % args[1]) + else: + move_number = None + try: + show_sgf_file(pathname, move_number) + except Exception, e: + print >>sys.stderr, "show_sgf:", str(e) + sys.exit(1) + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/gomill/examples/split_sgf_collection.py b/gomill/examples/split_sgf_collection.py new file mode 100644 index 0000000..3d55fcd --- /dev/null +++ b/gomill/examples/split_sgf_collection.py @@ -0,0 +1,54 @@ +"""Split an SGF collection into separate files. + +This demonstrates the parsing functions from the sgf_grammar module. + +""" + +import os +import sys +from optparse import OptionParser + +from gomill import sgf_grammar +from gomill import sgf + +def split_sgf_collection(pathname): + f = open(pathname) + sgf_src = f.read() + f.close() + dirname, basename = os.path.split(pathname) + root, ext = os.path.splitext(basename) + try: + coarse_games = sgf_grammar.parse_sgf_collection(sgf_src) + except ValueError, e: + raise StandardError("error parsing file: %s" % e) + for i, coarse_game in enumerate(coarse_games): + sgf_game = sgf.Sgf_game.from_coarse_game_tree(coarse_game) + sgf_game.get_root().add_comment_text( + "Split from %s (game %d)" % (basename, i+1)) + split_pathname = os.path.join(dirname, "%s_%d%s" % (root, i+1, ext)) + with open(split_pathname, "wb") as f: + f.write(sgf_game.serialise()) + + +_description = """\ +Split a file containing an SGF game collection into multiple files. +""" + +def main(argv): + parser = OptionParser(usage="%prog ", + description=_description) + opts, args = parser.parse_args(argv) + if not args: + parser.error("not enough arguments") + pathname = args[0] + if len(args) > 1: + parser.error("too many arguments") + try: + split_sgf_collection(pathname) + except Exception, e: + print >>sys.stderr, "sgf_splitter:", str(e) + sys.exit(1) + +if __name__ == "__main__": + main(sys.argv[1:]) + diff --git a/gomill/examples/twogtp b/gomill/examples/twogtp new file mode 100755 index 0000000..d97ed1d --- /dev/null +++ b/gomill/examples/twogtp @@ -0,0 +1,75 @@ +#!/usr/bin/env python +"""'Traditional' twogtp implementation. + +This runs a single game between two local GTP engines and reports the result. + +This demonstrates the gtp_games module. + +""" + +import shlex +import sys +from optparse import OptionParser, SUPPRESS_HELP + +from gomill import ascii_boards +from gomill import gtp_games +from gomill.gtp_controller import GtpChannelError, BadGtpResponse +from gomill.common import format_vertex + +def print_move(colour, move, board): + print colour.upper(), format_vertex(move) + +def print_board(colour, move, board): + print colour.upper(), format_vertex(move) + print ascii_boards.render_board(board) + print + +def main(): + usage = "%prog [options] --black='' --white=''" + parser = OptionParser(usage=usage) + parser.add_option("--black", help=SUPPRESS_HELP) + parser.add_option("--white", help=SUPPRESS_HELP) + parser.add_option("--komi", type="float", default=7.5) + parser.add_option("--size", type="int", default=19) + parser.add_option("--verbose", type="choice", choices=('0','1','2'), + default=0, metavar="0|1|2") + + (options, args) = parser.parse_args() + if args: + parser.error("too many arguments") + if not options.black or not options.white: + parser.error("players not specified") + + black_command = shlex.split(options.black) + white_command = shlex.split(options.white) + game = gtp_games.Game( + board_size=options.size, + komi=options.komi, + move_limit=1000) + if black_command[0] != white_command[0]: + game.set_player_code('b', black_command[0]) + game.set_player_code('w', white_command[0]) + if options.verbose == '1': + game.set_move_callback(print_move) + elif options.verbose == '2': + game.set_move_callback(print_board) + + game.allow_scorer('b') + game.allow_scorer('w') + + try: + game.set_player_subprocess('b', black_command) + game.set_player_subprocess('w', white_command) + game.ready() + game.run() + except (GtpChannelError, BadGtpResponse), e: + sys.exit("aborting game due to error:\n%s\n" % e) + finally: + game.close_players() + print game.result.describe() + if game.late_errors: + sys.exit("\n".join(game.late_errors)) + +if __name__ == "__main__": + main() + diff --git a/gomill/gomill/__init__.py b/gomill/gomill/__init__.py new file mode 100644 index 0000000..bc8c296 --- /dev/null +++ b/gomill/gomill/__init__.py @@ -0,0 +1 @@ +__version__ = "0.7.2" diff --git a/gomill/gomill/allplayalls.py b/gomill/gomill/allplayalls.py new file mode 100644 index 0000000..76019b8 --- /dev/null +++ b/gomill/gomill/allplayalls.py @@ -0,0 +1,257 @@ +"""Competitions for all-play-all tournaments.""" + +from gomill import ascii_tables +from gomill import game_jobs +from gomill import competitions +from gomill import tournaments +from gomill import tournament_results +from gomill.competitions import ( + Competition, CompetitionError, ControlFileError) +from gomill.settings import * +from gomill.utils import format_float + + +class Competitor_config(Quiet_config): + """Competitor description for use in control files.""" + # positional or keyword + positional_arguments = ('player',) + # keyword-only + keyword_arguments = () + +class Competitor_spec(object): + """Internal description of a competitor spec from the configuration file. + + Public attributes: + player -- player code + short_code -- eg 'A' or 'ZZ' + + """ + + +class Allplayall(tournaments.Tournament): + """A Tournament with matchups for all pairs of competitors. + + The game ids are like AvB_2, where A and B are the competitor short_codes + and 2 is the game number between those two competitors. + + This tournament type doesn't permit ghost matchups. + + """ + + def control_file_globals(self): + result = Competition.control_file_globals(self) + result.update({ + 'Competitor' : Competitor_config, + }) + return result + + + special_settings = [ + Setting('competitors', + interpret_sequence_of_quiet_configs( + Competitor_config, allow_simple_values=True)), + ] + + def competitor_spec_from_config(self, i, competitor_config): + """Make a Competitor_spec from a Competitor_config. + + i -- ordinal number of the competitor. + + Raises ControlFileError if there is an error in the configuration. + + Returns a Competitor_spec with all attributes set. + + """ + arguments = competitor_config.resolve_arguments() + cspec = Competitor_spec() + + if 'player' not in arguments: + raise ValueError("player not specified") + cspec.player = arguments['player'] + if cspec.player not in self.players: + raise ControlFileError("unknown player") + + def let(n): + return chr(ord('A') + n) + if i < 26: + cspec.short_code = let(i) + elif i < 26*27: + n, m = divmod(i, 26) + cspec.short_code = let(n-1) + let(m) + else: + raise ValueError("too many competitors") + return cspec + + @staticmethod + def _get_matchup_id(c1, c2): + return "%sv%s" % (c1.short_code, c2.short_code) + + def initialise_from_control_file(self, config): + Competition.initialise_from_control_file(self, config) + + matchup_settings = [ + setting for setting in competitions.game_settings + if setting.name not in ('handicap', 'handicap_style') + ] + [ + Setting('rounds', allow_none(interpret_int), default=None), + ] + try: + matchup_parameters = load_settings(matchup_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + matchup_parameters['alternating'] = True + matchup_parameters['number_of_games'] = matchup_parameters.pop('rounds') + + try: + specials = load_settings(self.special_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + + if not specials['competitors']: + raise ControlFileError("competitors: empty list") + # list of Competitor_specs + self.competitors = [] + seen_competitors = set() + for i, competitor_spec in enumerate(specials['competitors']): + try: + cspec = self.competitor_spec_from_config(i, competitor_spec) + except StandardError, e: + code = competitor_spec.get_key() + if code is None: + code = i + raise ControlFileError("competitor %s: %s" % (code, e)) + if cspec.player in seen_competitors: + raise ControlFileError("duplicate competitor: %s" + % cspec.player) + seen_competitors.add(cspec.player) + self.competitors.append(cspec) + + # map matchup_id -> Matchup + self.matchups = {} + # Matchups in order of definition + self.matchup_list = [] + for c1_i, c1 in enumerate(self.competitors): + for c2 in self.competitors[c1_i+1:]: + try: + m = self.make_matchup( + self._get_matchup_id(c1, c2), + c1.player, c2.player, + matchup_parameters) + except StandardError, e: + raise ControlFileError("%s v %s: %s" % + (c1.player, c2.player, e)) + self.matchups[m.id] = m + self.matchup_list.append(m) + + + # Can bump this to prevent people loading incompatible .status files. + status_format_version = 1 + + def get_status(self): + result = tournaments.Tournament.get_status(self) + result['competitors'] = [c.player for c in self.competitors] + return result + + def set_status(self, status): + seen_competitors = status['competitors'] + # This should mean that _check_results can never fail, but might as well + # still let it run. + if len(self.competitors) < len(seen_competitors): + raise CompetitionError( + "competitor has been removed from control file") + if ([c.player for c in self.competitors[:len(seen_competitors)]] != + seen_competitors): + raise CompetitionError( + "competitors have changed in the control file") + tournaments.Tournament.set_status(self, status) + + + def get_player_checks(self): + result = [] + matchup = self.matchup_list[0] + for competitor in self.competitors: + check = game_jobs.Player_check() + check.player = self.players[competitor.player] + check.board_size = matchup.board_size + check.komi = matchup.komi + result.append(check) + return result + + + def count_games_played(self): + """Return the total number of games completed.""" + return sum(len(l) for l in self.results.values()) + + def count_games_expected(self): + """Return the total number of games required. + + Returns None if no limit has been set. + + """ + rounds = self.matchup_list[0].number_of_games + if rounds is None: + return None + n = len(self.competitors) + return rounds * n * (n-1) // 2 + + def write_screen_report(self, out): + expected = self.count_games_expected() + if expected is not None: + print >>out, "%d/%d games played" % ( + self.count_games_played(), expected) + else: + print >>out, "%d games played" % self.count_games_played() + print >>out + + t = ascii_tables.Table(row_count=len(self.competitors)) + t.add_heading("") # player short_code + i = t.add_column(align='left') + t.set_column_values(i, (c.short_code for c in self.competitors)) + + t.add_heading("") # player code + i = t.add_column(align='left') + t.set_column_values(i, (c.player for c in self.competitors)) + + for c2_i, c2 in enumerate(self.competitors): + t.add_heading(" " + c2.short_code) + i = t.add_column(align='left') + column_values = [] + for c1_i, c1 in enumerate(self.competitors): + if c1_i == c2_i: + column_values.append("") + continue + if c1_i < c2_i: + matchup_id = self._get_matchup_id(c1, c2) + matchup = self.matchups[matchup_id] + player_x = matchup.player_1 + player_y = matchup.player_2 + else: + matchup_id = self._get_matchup_id(c2, c1) + matchup = self.matchups[matchup_id] + player_x = matchup.player_2 + player_y = matchup.player_1 + ms = tournament_results.Matchup_stats( + self.results[matchup.id], + player_x, player_y) + column_values.append( + "%s-%s" % (format_float(ms.wins_1), + format_float(ms.wins_2))) + t.set_column_values(i, column_values) + print >>out, "\n".join(t.render()) + + def write_short_report(self, out): + def p(s): + print >>out, s + p("allplayall: %s" % self.competition_code) + if self.description: + p(self.description) + p('') + self.write_screen_report(out) + p('') + self.write_matchup_reports(out) + p('') + self.write_player_descriptions(out) + p('') + + write_full_report = write_short_report + diff --git a/gomill/gomill/ascii_boards.py b/gomill/gomill/ascii_boards.py new file mode 100644 index 0000000..1bb79d0 --- /dev/null +++ b/gomill/gomill/ascii_boards.py @@ -0,0 +1,80 @@ +"""ASCII board representation.""" + +from gomill.common import * +from gomill import boards +from gomill.common import column_letters + +def render_grid(point_formatter, size): + """Render a board-shaped grid as a list of strings. + + point_formatter -- function (row, col) -> string of length 2. + + Returns a list of strings. + + """ + column_header_string = " ".join(column_letters[i] for i in range(size)) + result = [] + if size > 9: + rowstart = "%2d " + padding = " " + else: + rowstart = "%d " + padding = "" + for row in range(size-1, -1, -1): + result.append(rowstart % (row+1) + + " ".join(point_formatter(row, col) + for col in range(size))) + result.append(padding + " " + column_header_string) + return result + +_point_strings = { + None : " .", + 'b' : " #", + 'w' : " o", + } + +def render_board(board): + """Render a gomill Board in ascii. + + Returns a string without final newline. + + """ + def format_pt(row, col): + return _point_strings.get(board.get(row, col), " ?") + return "\n".join(render_grid(format_pt, board.side)) + +def interpret_diagram(diagram, size, board=None): + """Set up the position from a diagram. + + diagram -- board representation as from render_board() + size -- int + + Returns a Board. + + If the optional 'board' parameter is provided, it must be an empty board of + the right size; the same object will be returned. + + """ + if board is None: + board = boards.Board(size) + else: + if board.side != size: + raise ValueError("wrong board size, must be %d" % size) + if not board.is_empty(): + raise ValueError("board not empty") + lines = diagram.split("\n") + colours = {'#' : 'b', 'o' : 'w', '.' : None} + if size > 9: + extra_offset = 1 + else: + extra_offset = 0 + try: + for (row, col) in board.board_points: + colour = colours[lines[size-row-1][3*(col+1)+extra_offset]] + if colour is not None: + board.play(row, col, colour) + except Exception: + raise ValueError + return board + + diff --git a/gomill/gomill/ascii_tables.py b/gomill/gomill/ascii_tables.py new file mode 100644 index 0000000..0bf7590 --- /dev/null +++ b/gomill/gomill/ascii_tables.py @@ -0,0 +1,149 @@ +"""Render tabular output. + +This is designed for screen or text-file output, using a fixed-width font. + +""" + +from collections import defaultdict + +class Column_spec(object): + """Details of a table column. + + Public attributes: + align -- 'left' or 'right' + right_padding -- int + + """ + def __init__(self, align='left', right_padding=1): + self.align = align + self.right_padding = right_padding + + def render(self, s, width): + if self.align == 'left': + s = s.ljust(width) + elif self.align == 'right': + s = s.rjust(width) + return s + " " * self.right_padding + +class Table(object): + """Render tabular output. + + Normal use: + + tbl = Table(row_count=3) + tbl.add_heading('foo') + i = tbl.add_column(align='left', right_padding=3) + tbl.set_column_values(i, ['a', 'b']) + [...] + print '\n'.join(tbl.render()) + + """ + def __init__(self, row_count=None): + self.col_count = 0 + self.row_count = row_count + self.headings = [] + self.columns = [] + self.cells = defaultdict(str) + + def set_row_count(self, row_count): + """Change the table's row count.""" + self.row_count = row_count + + def add_heading(self, heading, span=1): + """Specify a column or column group heading. + + To leave a column with no heading, pass the empty string. + + To allow a heading to cover multiple columns, pass the 'span' parameter + and don't add headings for the rest of the covered columns. + + """ + self.headings.append((heading, span)) + + def add_column(self, **kwargs): + """Add a column to the table. + + align -- 'left' (default) or 'right' + right_padding -- int (default 1) + + Returns the column id + + Right padding is the number of spaces to leave between this column and + the next. + + (The last column should have right padding 1, so that the heading can + use the full width if necessary.) + + """ + column = Column_spec(**kwargs) + self.columns.append(column) + column_id = self.col_count + self.col_count += 1 + return column_id + + def get_column(self, column_id): + """Retrieve a column object given its id. + + You can use this to change the column's attributes after adding it. + + """ + return self.columns[column_id] + + def set_column_values(self, column_id, values): + """Specify the values for a column. + + column_id -- as returned by add_column() + values -- iterable + + str() is called on the values. + + If values are not supplied for all rows, the remaining rows are left + blank. If too many values are supplied, the excess values are ignored. + + """ + for row, value in enumerate(values): + self.cells[row, column_id] = str(value) + + def render(self): + """Render the table. + + Returns a list of strings. + + Each line has no trailing whitespace. + + Lines which would be wholly blank are omitted. + + """ + def column_values(col): + return [self.cells[row, col] for row in xrange(self.row_count)] + + result = [] + + cells = self.cells + widths = [max(map(len, column_values(i))) + for i in xrange(self.col_count)] + col = 0 + heading_line = [] + for heading, span in self.headings: + # width available for the heading + width = (sum(widths[col:col+span]) + + sum(self.columns[i].right_padding + for i in range(col, col+span)) - 1) + shortfall = len(heading) - width + if shortfall > 0: + width += shortfall + # Make the leftmost column in the span wider to fit the heading + widths[col] += shortfall + heading_line.append(heading.ljust(width)) + col += span + result.append(" ".join(heading_line).rstrip()) + + for row in xrange(self.row_count): + l = [] + for col, (column, width) in enumerate(zip(self.columns, widths)): + l.append(column.render(cells[row, col], width)) + line = "".join(l).rstrip() + if line: + result.append(line) + return result + diff --git a/gomill/gomill/boards.py b/gomill/gomill/boards.py new file mode 100644 index 0000000..d264fed --- /dev/null +++ b/gomill/gomill/boards.py @@ -0,0 +1,248 @@ +"""Go board representation.""" + +from gomill.common import * + + +class _Group(object): + """Represent a solidly-connected group. + + Public attributes: + colour + points + is_surrounded + + Points are coordinate pairs (row, col). + + """ + +class _Region(object): + """Represent an empty region. + + Public attributes: + points + neighbouring_colours + + Points are coordinate pairs (row, col). + + """ + def __init__(self): + self.points = set() + self.neighbouring_colours = set() + +class Board(object): + """A legal Go position. + + Supports playing stones with captures, and area scoring. + + Public attributes: + side -- board size (eg 9) + board_points -- list of coordinates of all points on the board + + Behaviour is unspecified if methods are passed out-of-range coordinates. + + """ + def __init__(self, side): + self.side = side + self.board_points = [(_row, _col) for _row in range(side) + for _col in range(side)] + self.board = [] + for row in range(side): + self.board.append([None] * side) + self._is_empty = True + + def copy(self): + """Return an independent copy of this Board.""" + b = Board(self.side) + b.board = [self.board[i][:] for i in xrange(self.side)] + b._is_empty = self._is_empty + return b + + def _make_group(self, row, col, colour): + points = set() + is_surrounded = True + to_handle = set() + to_handle.add((row, col)) + while to_handle: + point = to_handle.pop() + points.add(point) + r, c = point + for neighbour in [(r-1, c), (r+1, c), (r, c-1), (r, c+1)]: + (r1, c1) = neighbour + if not ((0 <= r1 < self.side) and (0 <= c1 < self.side)): + continue + neigh_colour = self.board[r1][c1] + if neigh_colour is None: + is_surrounded = False + elif neigh_colour == colour: + if neighbour not in points: + to_handle.add(neighbour) + group = _Group() + group.colour = colour + group.points = points + group.is_surrounded = is_surrounded + return group + + def _make_empty_region(self, row, col): + points = set() + neighbouring_colours = set() + to_handle = set() + to_handle.add((row, col)) + while to_handle: + point = to_handle.pop() + points.add(point) + r, c = point + for neighbour in [(r-1, c), (r+1, c), (r, c-1), (r, c+1)]: + (r1, c1) = neighbour + if not ((0 <= r1 < self.side) and (0 <= c1 < self.side)): + continue + neigh_colour = self.board[r1][c1] + if neigh_colour is None: + if neighbour not in points: + to_handle.add(neighbour) + else: + neighbouring_colours.add(neigh_colour) + region = _Region() + region.points = points + region.neighbouring_colours = neighbouring_colours + return region + + def _find_surrounded_groups(self): + """Find solidly-connected groups with 0 liberties. + + Returns a list of _Groups. + + """ + surrounded = [] + handled = set() + for (row, col) in self.board_points: + colour = self.board[row][col] + if colour is None: + continue + point = (row, col) + if point in handled: + continue + group = self._make_group(row, col, colour) + if group.is_surrounded: + surrounded.append(group) + handled.update(group.points) + return surrounded + + def is_empty(self): + """Say whether the board is empty.""" + return self._is_empty + + def get(self, row, col): + """Return the state of the specified point. + + Returns a colour, or None for an empty point. + + """ + return self.board[row][col] + + def play(self, row, col, colour): + """Play a move on the board. + + Raises ValueError if the specified point isn't empty. + + Performs any necessary captures. Allows self-captures. Doesn't enforce + any ko rule. + + Returns the point forbidden by simple ko, or None + + """ + if self.board[row][col] is not None: + raise ValueError + self.board[row][col] = colour + surrounded = self._find_surrounded_groups() + simple_ko_point = None + if surrounded: + if len(surrounded) == 1: + to_capture = surrounded + else: + to_capture = [group for group in surrounded + if group.colour == opponent_of(colour)] + if len(to_capture) == 1 and len(to_capture[0].points) == 1: + self_capture = [group for group in surrounded + if group.colour == colour] + if len(self_capture[0].points) == 1: + simple_ko_point = iter(to_capture[0].points).next() + for group in to_capture: + for r, c in group.points: + self.board[r][c] = None + self._is_empty = False + return simple_ko_point + + def apply_setup(self, black_points, white_points, empty_points): + """Add setup stones or removals to the position. + + This is intended to support SGF AB/AW/AE commands. + + Each parameter is an iterable of coordinate pairs (row, col). + + Applies all the setup specifications, then removes any groups with no + liberties (so the resulting position is always legal). + + If the same point is specified in more than one list, the order in which + they're applied is undefined. + + Returns a boolean saying whether the position was legal as specified. + + """ + for (row, col) in black_points: + self.board[row][col] = 'b' + for (row, col) in white_points: + self.board[row][col] = 'w' + for (row, col) in empty_points: + self.board[row][col] = None + captured = self._find_surrounded_groups() + for group in captured: + for row, col in group.points: + self.board[row][col] = None + self._is_empty = True + for (row, col) in self.board_points: + if self.board[row][col] is not None: + self._is_empty = False + break + return not(captured) + + def list_occupied_points(self): + """List all nonempty points. + + Returns a list of pairs (colour, (row, col)) + + """ + result = [] + for (row, col) in self.board_points: + colour = self.board[row][col] + if colour is not None: + result.append((colour, (row, col))) + return result + + def area_score(self): + """Calculate the area score of a position. + + Assumes all stones are alive. + + Returns black score minus white score. + + Doesn't take komi into account. + + """ + scores = {'b' : 0, 'w' : 0} + handled = set() + for (row, col) in self.board_points: + colour = self.board[row][col] + if colour is not None: + scores[colour] += 1 + continue + point = (row, col) + if point in handled: + continue + region = self._make_empty_region(row, col) + region_size = len(region.points) + for colour in ('b', 'w'): + if colour in region.neighbouring_colours: + scores[colour] += region_size + handled.update(region.points) + return scores['b'] - scores['w'] + diff --git a/gomill/gomill/cem_tuners.py b/gomill/gomill/cem_tuners.py new file mode 100644 index 0000000..41ebfb7 --- /dev/null +++ b/gomill/gomill/cem_tuners.py @@ -0,0 +1,509 @@ +"""Competitions for parameter tuning using the cross-entropy method.""" + +from __future__ import division + +from random import gauss as random_gauss +from math import sqrt + +from gomill import compact_tracebacks +from gomill import game_jobs +from gomill import competitions +from gomill import competition_schedulers +from gomill.competitions import ( + Competition, NoGameAvailable, CompetitionError, ControlFileError, + Player_config) +from gomill.settings import * + + +def square(f): + return f * f + +class Distribution(object): + """A multi-dimensional Gaussian probability distribution. + + Instantiate with a list of pairs of floats (mean, variance) + + Public attributes: + parameters -- the list used to instantiate the distribution + + """ + def __init__(self, parameters): + self.dimension = len(parameters) + if self.dimension == 0: + raise ValueError + self.parameters = parameters + self.gaussian_params = [(mean, sqrt(variance)) + for (mean, variance) in parameters] + + def get_sample(self): + """Return a random sample from the distribution. + + Returns a list of floats + + """ + return [random_gauss(mean, stddev) + for (mean, stddev) in self.gaussian_params] + + def get_means(self): + """Return just the mean from each dimension. + + Returns a list of floats. + + """ + return [mean for (mean, stddev) in self.parameters] + + def format(self): + return " ".join("%5.2f~%4.2f" % (mean, stddev) + for (mean, stddev) in self.parameters) + + def __str__(self): + return "" % self.format() + +def update_distribution(distribution, elites, step_size): + """Update a distribution based on the given elites. + + distribution -- Distribution + elites -- list of optimiser parameter vectors + step_size -- float between 0.0 and 1.0 ('alpha') + + Returns a new distribution + + """ + n = len(elites) + new_distribution_parameters = [] + for i in range(distribution.dimension): + v = [e[i] for e in elites] + elite_mean = sum(v) / n + elite_var = sum(map(square, v)) / n - square(elite_mean) + old_mean, old_var = distribution.parameters[i] + new_mean = (elite_mean * step_size + + old_mean * (1.0 - step_size)) + new_var = (elite_var * step_size + + old_var * (1.0 - step_size)) + new_distribution_parameters.append((new_mean, new_var)) + return Distribution(new_distribution_parameters) + + +parameter_settings = [ + Setting('code', interpret_identifier), + Setting('initial_mean', interpret_float), + Setting('initial_variance', interpret_float), + Setting('transform', interpret_callable, default=float), + Setting('format', interpret_8bit_string, default=None), + ] + +class Parameter_config(Quiet_config): + """Parameter (ie, dimension) description for use in control files.""" + # positional or keyword + positional_arguments = ('code',) + # keyword-only + keyword_arguments = tuple(setting.name for setting in parameter_settings + if setting.name != 'code') + +class Parameter_spec(object): + """Internal description of a parameter spec from the configuration file. + + Public attributes: + code -- identifier + initial_mean -- float + initial_variance -- float + transform -- function float -> player parameter + format -- string for use with '%' + + """ + +class Cem_tuner(Competition): + """A Competition for parameter tuning using the cross-entropy method. + + The game ids are like 'g0#1r3', where 0 is the generation number, 1 is the + candidate number and 3 is the round number. + + """ + def __init__(self, competition_code, **kwargs): + Competition.__init__(self, competition_code, **kwargs) + self.seen_successful_game = False + + def control_file_globals(self): + result = Competition.control_file_globals(self) + result.update({ + 'Parameter' : Parameter_config, + }) + return result + + global_settings = (Competition.global_settings + + competitions.game_settings + [ + Setting('batch_size', interpret_positive_int), + Setting('samples_per_generation', interpret_positive_int), + Setting('number_of_generations', interpret_positive_int), + Setting('elite_proportion', interpret_float), + Setting('step_size', interpret_float), + ]) + + special_settings = [ + Setting('opponent', interpret_identifier), + Setting('parameters', + interpret_sequence_of_quiet_configs(Parameter_config)), + Setting('make_candidate', interpret_callable), + ] + + def parameter_spec_from_config(self, parameter_config): + """Make a Parameter_spec from a Parameter_config. + + Raises ControlFileError if there is an error in the configuration. + + Returns a Parameter_spec with all attributes set. + + """ + if not isinstance(parameter_config, Parameter_config): + raise ControlFileError("not a Parameter") + + arguments = parameter_config.resolve_arguments() + interpreted = load_settings(parameter_settings, arguments) + pspec = Parameter_spec() + for name, value in interpreted.iteritems(): + setattr(pspec, name, value) + if pspec.initial_variance < 0.0: + raise ValueError("'initial_variance': must be nonnegative") + try: + transformed = pspec.transform(pspec.initial_mean) + except Exception: + raise ValueError( + "error from transform (applied to initial_mean)\n%s" % + (compact_tracebacks.format_traceback(skip=1))) + if pspec.format is None: + pspec.format = pspec.code + ":%s" + try: + pspec.format % transformed + except Exception: + raise ControlFileError("'format': invalid format string") + return pspec + + def initialise_from_control_file(self, config): + Competition.initialise_from_control_file(self, config) + + competitions.validate_handicap( + self.handicap, self.handicap_style, self.board_size) + + if not 0.0 < self.elite_proportion < 1.0: + raise ControlFileError("elite_proportion out of range (0.0 to 1.0)") + if not 0.0 < self.step_size < 1.0: + raise ControlFileError("step_size out of range (0.0 to 1.0)") + + try: + specials = load_settings(self.special_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + + try: + self.opponent = self.players[specials['opponent']] + except KeyError: + raise ControlFileError( + "opponent: unknown player %s" % specials['opponent']) + + self.parameter_specs = [] + if not specials['parameters']: + raise ControlFileError("parameters: empty list") + seen_codes = set() + for i, parameter_spec in enumerate(specials['parameters']): + try: + pspec = self.parameter_spec_from_config(parameter_spec) + except StandardError, e: + code = parameter_spec.get_key() + if code is None: + code = i + raise ControlFileError("parameter %s: %s" % (code, e)) + if pspec.code in seen_codes: + raise ControlFileError( + "duplicate parameter code: %s" % pspec.code) + seen_codes.add(pspec.code) + self.parameter_specs.append(pspec) + + self.candidate_maker_fn = specials['make_candidate'] + + self.initial_distribution = Distribution( + [(pspec.initial_mean, pspec.initial_variance) + for pspec in self.parameter_specs]) + + + # State attributes (*: in persistent state): + # *generation -- current generation (0-based int) + # *distribution -- Distribution for current generation + # *sample_parameters -- optimiser_params + # (list indexed by candidate number) + # *wins -- number of games won + # half a point for a game with no winner + # (list indexed by candidate number) + # candidates -- Players (code attribute is the candidate code) + # (list indexed by candidate number) + # *scheduler -- Group_scheduler (group codes are candidate numbers) + # + # These are all reset for each new generation. + # + # seen_successful_game -- bool (per-run state) + + def set_clean_status(self): + self.generation = 0 + self.distribution = self.initial_distribution + self.reset_for_new_generation() + + def _set_scheduler_groups(self): + self.scheduler.set_groups( + (i, self.batch_size) for i in xrange(self.samples_per_generation) + ) + + # Can bump this to prevent people loading incompatible .status files. + status_format_version = 0 + + def get_status(self): + return { + 'generation' : self.generation, + 'distribution' : self.distribution.parameters, + 'sample_parameters' : self.sample_parameters, + 'wins' : self.wins, + 'scheduler' : self.scheduler, + } + + def set_status(self, status): + self.generation = status['generation'] + self.distribution = Distribution(status['distribution']) + self.sample_parameters = status['sample_parameters'] + self.wins = status['wins'] + self.prepare_candidates() + self.scheduler = status['scheduler'] + # Might as well notice if they changed the batch_size + self._set_scheduler_groups() + self.scheduler.rollback() + + def reset_for_new_generation(self): + get_sample = self.distribution.get_sample + self.sample_parameters = [get_sample() + for _ in xrange(self.samples_per_generation)] + self.wins = [0] * self.samples_per_generation + self.prepare_candidates() + self.scheduler = competition_schedulers.Group_scheduler() + self._set_scheduler_groups() + + def transform_parameters(self, optimiser_parameters): + l = [] + for pspec, v in zip(self.parameter_specs, optimiser_parameters): + try: + l.append(pspec.transform(v)) + except Exception: + raise CompetitionError( + "error from transform for %s\n%s" % + (pspec.code, compact_tracebacks.format_traceback(skip=1))) + return tuple(l) + + def format_engine_parameters(self, engine_parameters): + l = [] + for pspec, v in zip(self.parameter_specs, engine_parameters): + try: + s = pspec.format % v + except Exception: + s = "[%s?%s]" % (pspec.code, v) + l.append(s) + return "; ".join(l) + + def format_optimiser_parameters(self, optimiser_parameters): + return self.format_engine_parameters(self.transform_parameters( + optimiser_parameters)) + + @staticmethod + def make_candidate_code(generation, candidate_number): + return "g%d#%d" % (generation, candidate_number) + + def make_candidate(self, player_code, engine_parameters): + """Make a player using the specified engine parameters. + + Returns a game_jobs.Player. + + """ + try: + candidate_config = self.candidate_maker_fn(*engine_parameters) + except Exception: + raise CompetitionError( + "error from make_candidate()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + if not isinstance(candidate_config, Player_config): + raise CompetitionError( + "make_candidate() returned %r, not Player" % + candidate_config) + try: + candidate = self.game_jobs_player_from_config( + player_code, candidate_config) + except Exception, e: + raise CompetitionError( + "bad player spec from make_candidate():\n" + "%s\nparameters were: %s" % + (e, self.format_engine_parameters(engine_parameters))) + return candidate + + def prepare_candidates(self): + """Set up the candidates array. + + This is run for each new generation, and when reloading state. + + Requires generation and sample_parameters to be already set. + + Initialises self.candidates. + + """ + self.candidates = [] + for candidate_number, optimiser_params in \ + enumerate(self.sample_parameters): + candidate_code = self.make_candidate_code( + self.generation, candidate_number) + engine_parameters = self.transform_parameters(optimiser_params) + self.candidates.append( + self.make_candidate(candidate_code, engine_parameters)) + + def finish_generation(self): + """Process a generation's results and calculate the new distribution. + + Writes a description of the generation to the history log. + + Updates self.distribution. + + """ + sorter = [(wins, candidate_number) + for (candidate_number, wins) in enumerate(self.wins)] + sorter.sort(reverse=True) + elite_count = max(1, + int(self.elite_proportion * self.samples_per_generation + 0.5)) + self.log_history("Generation %s" % self.generation) + self.log_history("Distribution\n%s" % + self.format_distribution(self.distribution)) + self.log_history(self.format_generation_results(sorter, elite_count)) + self.log_history("") + elite_samples = [self.sample_parameters[index] + for (wins, index) in sorter[:elite_count]] + self.distribution = update_distribution( + self.distribution, elite_samples, self.step_size) + + def get_player_checks(self): + engine_parameters = self.transform_parameters( + self.initial_distribution.get_sample()) + candidate = self.make_candidate('candidate', engine_parameters) + result = [] + for player in [candidate, self.opponent]: + check = game_jobs.Player_check() + check.player = player + check.board_size = self.board_size + check.komi = self.komi + result.append(check) + return result + + def get_game(self): + if self.scheduler.nothing_issued_yet(): + self.log_event("\nstarting generation %d" % self.generation) + + candidate_number, round_id = self.scheduler.issue() + if candidate_number is None: + return NoGameAvailable + + candidate = self.candidates[candidate_number] + + job = game_jobs.Game_job() + job.game_id = "%sr%d" % (candidate.code, round_id) + job.game_data = (candidate_number, candidate.code, round_id) + job.player_b = candidate + job.player_w = self.opponent + job.board_size = self.board_size + job.komi = self.komi + job.move_limit = self.move_limit + job.handicap = self.handicap + job.handicap_is_free = (self.handicap_style == 'free') + job.use_internal_scorer = (self.scorer == 'internal') + job.internal_scorer_handicap_compensation = \ + self.internal_scorer_handicap_compensation + job.sgf_event = self.competition_code + job.sgf_note = ("Candidate parameters: %s" % + self.format_optimiser_parameters( + self.sample_parameters[candidate_number])) + return job + + def process_game_result(self, response): + self.seen_successful_game = True + candidate_number, candidate_code, round_id = response.game_data + self.scheduler.fix(candidate_number, round_id) + gr = response.game_result + assert candidate_code in (gr.player_b, gr.player_w) + + # Counting jigo or no-result as half a point for the candidate + if gr.winning_player == candidate_code: + self.wins[candidate_number] += 1 + elif gr.winning_player is None: + self.wins[candidate_number] += 0.5 + + if self.scheduler.all_fixed(): + self.finish_generation() + self.generation += 1 + if self.generation != self.number_of_generations: + self.reset_for_new_generation() + + def process_game_error(self, job, previous_error_count): + ## If the very first game to return a response gives an error, halt. + ## Otherwise, retry once and halt on a second failure. + stop_competition = False + retry_game = False + if (not self.seen_successful_game) or (previous_error_count > 0): + stop_competition = True + else: + retry_game = True + return stop_competition, retry_game + + + def format_distribution(self, distribution): + """Pretty-print a distribution. + + Returns a string. + + """ + return "%s\n%s" % ( + self.format_optimiser_parameters(distribution.get_means()), + distribution.format()) + + def format_generation_results(self, ordered_samples, elite_count): + """Pretty-print the results of a single generation. + + ordered_samples -- list of pairs (wins, candidate number) + elite_count -- number of samples to mark as elite + + """ + result = [] + for i, (wins, candidate_number) in enumerate(ordered_samples): + opt_parameters = self.sample_parameters[candidate_number] + result.append( + "%s%s %s %3d" % + (self.make_candidate_code(self.generation, candidate_number), + "*" if i < elite_count else " ", + self.format_optimiser_parameters(opt_parameters), + wins)) + return "\n".join(result) + + def write_static_description(self, out): + def p(s): + print >>out, s + p("CEM tuning event: %s" % self.competition_code) + if self.description: + p(self.description) + p("board size: %s" % self.board_size) + p("komi: %s" % self.komi) + + def write_screen_report(self, out): + print >>out, "generation %d" % self.generation + print >>out + print >>out, "wins from current samples:\n%s" % self.wins + print >>out + if self.generation == self.number_of_generations: + print >>out, "final distribution:" + else: + print >>out, "distribution for generation %d:" % self.generation + print >>out, self.format_distribution(self.distribution) + + def write_short_report(self, out): + self.write_static_description(out) + self.write_screen_report(out) + + write_full_report = write_short_report + diff --git a/gomill/gomill/common.py b/gomill/gomill/common.py new file mode 100644 index 0000000..71a7ff8 --- /dev/null +++ b/gomill/gomill/common.py @@ -0,0 +1,92 @@ +"""Domain-dependent utility functions for gomill. + +This module is designed to be used with 'from common import *'. + +This is for Go-specific utilities; see utils for generic utility functions. + +""" + +__all__ = ["opponent_of", "colour_name", "format_vertex", "format_vertex_list", + "move_from_vertex"] + +_opponents = {"b":"w", "w":"b"} +def opponent_of(colour): + """Return the opponent colour. + + colour -- 'b' or 'w' + + Returns 'b' or 'w'. + + """ + try: + return _opponents[colour] + except KeyError: + raise ValueError + +def colour_name(colour): + """Return the (lower-case) full name of a colour. + + colour -- 'b' or 'w' + + """ + try: + return {'b': 'black', 'w': 'white'}[colour] + except KeyError: + raise ValueError + + +column_letters = "ABCDEFGHJKLMNOPQRSTUVWXYZ" + +def format_vertex(move): + """Return coordinates as a string like 'A1', or 'pass'. + + move -- pair (row, col), or None for a pass + + The result is suitable for use directly in GTP responses. + + """ + if move is None: + return "pass" + row, col = move + if not 0 <= row < 25 or not 0 <= col < 25: + raise ValueError + return column_letters[col] + str(row+1) + +def format_vertex_list(moves): + """Return a list of coordinates as a string like 'A1,B2'.""" + return ",".join(map(format_vertex, moves)) + +def move_from_vertex(vertex, board_size): + """Interpret a string representing a vertex, as specified by GTP. + + Returns a pair of coordinates (row, col) in range(0, board_size) + + Raises ValueError with an appropriate message if 'vertex' isn't a valid GTP + vertex specification for a board of size 'board_size'. + + """ + if not 0 < board_size <= 25: + raise ValueError("board_size out of range") + try: + s = vertex.lower() + except Exception: + raise ValueError("invalid vertex") + if s == "pass": + return None + try: + col_c = s[0] + if (not "a" <= col_c <= "z") or col_c == "i": + raise ValueError + if col_c > "i": + col = ord(col_c) - ord("b") + else: + col = ord(col_c) - ord("a") + row = int(s[1:]) - 1 + if row < 0: + raise ValueError + except (IndexError, ValueError): + raise ValueError("invalid vertex: '%s'" % s) + if not (col < board_size and row < board_size): + raise ValueError("vertex is off board: '%s'" % s) + return row, col + diff --git a/gomill/gomill/compact_tracebacks.py b/gomill/gomill/compact_tracebacks.py new file mode 100644 index 0000000..a0a5cf1 --- /dev/null +++ b/gomill/gomill/compact_tracebacks.py @@ -0,0 +1,97 @@ +"""Compact formatting of tracebacks.""" + +import sys +import traceback + +def log_traceback_from_info(exception_type, value, tb, dst=sys.stderr, skip=0): + """Log a given exception nicely to 'dst', showing a traceback. + + dst -- writeable file-like object + skip -- number of traceback entries to omit from the top of the list + + """ + for line in traceback.format_exception_only(exception_type, value): + dst.write(line) + if (not isinstance(exception_type, str) and + issubclass(exception_type, SyntaxError)): + return + print >>dst, 'traceback (most recent call last):' + text = None + for filename, lineno, fnname, text in traceback.extract_tb(tb)[skip:]: + if fnname == "?": + fn_s = "" + else: + fn_s = "(%s)" % fnname + print >>dst, " %s:%s %s" % (filename, lineno, fn_s) + if text is not None: + print >>dst, "failing line:" + print >>dst, text + +def format_traceback_from_info(exception_type, value, tb, skip=0): + """Return a description of a given exception as a string. + + skip -- number of traceback entries to omit from the top of the list + + """ + from cStringIO import StringIO + log = StringIO() + log_traceback_from_info(exception_type, value, tb, log, skip) + return log.getvalue() + +def log_traceback(dst=sys.stderr, skip=0): + """Log the current exception nicely to 'dst'. + + dst -- writeable file-like object + skip -- number of traceback entries to omit from the top of the list + + """ + exception_type, value, tb = sys.exc_info() + log_traceback_from_info(exception_type, value, tb, dst, skip) + +def format_traceback(skip=0): + """Return a description of the current exception as a string. + + skip -- number of traceback entries to omit from the top of the list + + """ + exception_type, value, tb = sys.exc_info() + return format_traceback_from_info(exception_type, value, tb, skip) + + +def log_error_and_line_from_info(exception_type, value, tb, dst=sys.stderr): + """Log a given exception briefly to 'dst', showing line number.""" + if (not isinstance(exception_type, str) and + issubclass(exception_type, SyntaxError)): + for line in traceback.format_exception_only(exception_type, value): + dst.write(line) + else: + try: + filename, lineno, fnname, text = traceback.extract_tb(tb)[-1] + except IndexError: + pass + else: + print >>dst, "at line %s:" % lineno + for line in traceback.format_exception_only(exception_type, value): + dst.write(line) + +def format_error_and_line_from_info(exception_type, value, tb): + """Return a brief description of a given exception as a string.""" + from cStringIO import StringIO + log = StringIO() + log_error_and_line_from_info(exception_type, value, tb, log) + return log.getvalue() + +def log_error_and_line(dst=sys.stderr): + """Log the current exception briefly to 'dst'. + + dst -- writeable file-like object + + """ + exception_type, value, tb = sys.exc_info() + log_error_and_line_from_info(exception_type, value, tb, dst) + +def format_error_and_line(): + """Return a brief description of the current exception as a string.""" + exception_type, value, tb = sys.exc_info() + return format_error_and_line_from_info(exception_type, value, tb) + diff --git a/gomill/gomill/competition_schedulers.py b/gomill/gomill/competition_schedulers.py new file mode 100644 index 0000000..4af0f76 --- /dev/null +++ b/gomill/gomill/competition_schedulers.py @@ -0,0 +1,167 @@ +"""Schedule games in competitions. + +These schedulers are used to keep track of the ids of games which have been +started, and which have reported their results. + +They provide a mechanism to reissue ids of games which were in progress when an +unclean shutdown occurred. + +All scheduler classes are suitable for pickling. + +""" + +class Simple_scheduler(object): + """Schedule a single sequence of games. + + The issued tokens are integers counting up from zero. + + Public attributes (treat as read-only): + issued -- int + fixed -- int + + """ + def __init__(self): + self.next_new = 0 + self.outstanding = set() + self.to_reissue = set() + self.issued = 0 + self.fixed = 0 + #self._check_consistent() + + def _check_consistent(self): + assert self.issued == \ + self.next_new - len(self.to_reissue) + assert self.fixed == \ + self.next_new - len(self.outstanding) - len(self.to_reissue) + + def __getstate__(self): + return (self.next_new, self.outstanding, self.to_reissue) + + def __setstate__(self, state): + (self.next_new, self.outstanding, self.to_reissue) = state + self.issued = self.next_new - len(self.to_reissue) + self.fixed = self.issued - len(self.outstanding) + #self._check_consistent() + + def issue(self): + """Choose the next game to start. + + Returns an integer 'token'. + + """ + if self.to_reissue: + result = min(self.to_reissue) + self.to_reissue.discard(result) + else: + result = self.next_new + self.next_new += 1 + self.outstanding.add(result) + self.issued += 1 + #self._check_consistent() + return result + + def fix(self, token): + """Note that a game's result has been reliably stored.""" + self.outstanding.remove(token) + self.fixed += 1 + #self._check_consistent() + + def rollback(self): + """Make issued-but-not-fixed tokens available again.""" + self.issued -= len(self.outstanding) + self.to_reissue.update(self.outstanding) + self.outstanding = set() + #self._check_consistent() + + +class Group_scheduler(object): + """Schedule multiple lists of games in parallel. + + This schedules for a number of _groups_, each of which may have a limit on + the number of games to play. It schedules from the group (of those which + haven't reached their limit) with the fewest issued games, with smallest + group code breaking ties. + + group codes might be ints or short strings + (any sortable, pickleable and hashable object should do). + + The issued tokens are pairs (group code, game number), with game numbers + counting up from 0 independently for each group code. + + """ + def __init__(self): + self.allocators = {} + self.limits = {} + + def __getstate__(self): + return (self.allocators, self.limits) + + def __setstate__(self, state): + (self.allocators, self.limits) = state + + def set_groups(self, group_specs): + """Set the groups to be scheduled. + + group_specs -- iterable of pairs (group code, limit) + limit -- int or None + + You can call this again after the first time. The limits will be set to + the new values. Any existing groups not in the list are forgotten. + + """ + new_allocators = {} + new_limits = {} + for group_code, limit in group_specs: + if group_code in self.allocators: + new_allocators[group_code] = self.allocators[group_code] + else: + new_allocators[group_code] = Simple_scheduler() + new_limits[group_code] = limit + self.allocators = new_allocators + self.limits = new_limits + + def issue(self): + """Choose the next game to start. + + Returns a pair (group code, game number) + + Returns (None, None) if all groups have reached their limit. + + """ + groups = [ + (group_code, allocator.issued, self.limits[group_code]) + for (group_code, allocator) in self.allocators.iteritems() + ] + available = [ + (issue_count, group_code) + for (group_code, issue_count, limit) in groups + if limit is None or issue_count < limit + ] + if not available: + return None, None + _, group_code = min(available) + return group_code, self.allocators[group_code].issue() + + def fix(self, group_code, game_number): + """Note that a game's result has been reliably stored.""" + self.allocators[group_code].fix(game_number) + + def rollback(self): + """Make issued-but-not-fixed tokens available again.""" + for allocator in self.allocators.itervalues(): + allocator.rollback() + + def nothing_issued_yet(self): + """Say whether nothing has been issued yet.""" + return all(allocator.issued == 0 + for allocator in self.allocators.itervalues()) + + def all_fixed(self): + """Check whether all groups have reached their limits. + + This returns true if all groups have limits, and each group has as many + _fixed_ tokens as its limit. + + """ + return all(allocator.fixed >= self.limits[g] + for (g, allocator) in self.allocators.iteritems()) diff --git a/gomill/gomill/competitions.py b/gomill/gomill/competitions.py new file mode 100644 index 0000000..8e34e18 --- /dev/null +++ b/gomill/gomill/competitions.py @@ -0,0 +1,513 @@ +"""Organise processing jobs based around playing many GTP games.""" + +import os + +from gomill import game_jobs +from gomill import gtp_controller +from gomill import handicap_layout +from gomill.settings import * + + +def log_discard(s): + pass + +NoGameAvailable = object() + +class CompetitionError(StandardError): + """Error from competition code. + + This is intended for errors from user-provided functions, but it might also + indicate a bug in tuner code. + + The ringmaster should display the error and terminate immediately. + + """ + +class ControlFileError(StandardError): + """Error interpreting the control file.""" + + +class Control_file_token(object): + def __init__(self, name): + self.name = name + def __repr__(self): + return "<%s>" % self.name + + +_player_settings = [ + Setting('command', interpret_shlex_sequence), + Setting('cwd', allow_none(interpret_8bit_string), default=None), + Setting('environ', + allow_none(interpret_map_of( + interpret_8bit_string, interpret_8bit_string)), + default=None), + Setting('is_reliable_scorer', interpret_bool, default=True), + Setting('allow_claim', interpret_bool, default=False), + Setting('gtp_aliases', + allow_none(interpret_map_of( + interpret_8bit_string, interpret_8bit_string)), + defaultmaker=dict), + Setting('startup_gtp_commands', allow_none(interpret_sequence), + defaultmaker=list), + Setting('discard_stderr', interpret_bool, default=False), + ] + +class Player_config(Quiet_config): + """Player description for use in control files.""" + # positional or keyword + positional_arguments = ('command',) + # keyword-only + keyword_arguments = tuple(setting.name for setting in _player_settings) + + +class Competition(object): + """A resumable processing job based around playing many GTP games. + + This is an abstract base class. + + """ + + def __init__(self, competition_code): + self.competition_code = competition_code + self.base_directory = None + self.event_logger = log_discard + self.history_logger = log_discard + + def control_file_globals(self): + """Specify names and values to make available to the control file. + + Returns a dict suitable for use as the control file's namespace. + + """ + return { + 'Player' : Player_config, + } + + def set_base_directory(self, pathname): + """Set the competition's base directory. + + Relative paths in the control file are interpreted relative to this + directory. + + """ + self.base_directory = pathname + + def resolve_pathname(self, pathname): + """Resolve a pathname relative to the competition's base directory. + + Accepts None, returning it. + + Applies os.expanduser to the pathname. + + Doesn't absolutise or normalise the resulting pathname. + + Raises ValueError if it can't handle the pathname. + + """ + if pathname is None: + return None + if pathname == "": + raise ValueError("empty pathname") + try: + pathname = os.path.expanduser(pathname) + except Exception: + raise ValueError("bad pathname") + try: + return os.path.join(self.base_directory, pathname) + except Exception: + raise ValueError( + "relative path supplied but base directory isn't set") + + def set_event_logger(self, logger): + """Set a callback for the event log. + + logger -- function taking a string argument + + Until this is called, event log output is silently discarded. + + """ + self.event_logger = logger + + def set_history_logger(self, logger): + """Set a callback for the history file. + + logger -- function taking a string argument + + Until this is called, event log output is silently discarded. + + """ + self.history_logger = logger + + def log_event(self, s): + """Write a message to the event log. + + The event log logs all game starts and finishes; competitions can add + lines to mark things like the start of new generations. + + A newline is added to the message. + + """ + self.event_logger(s) + + def log_history(self, s): + """Write a message to the history file. + + The history file is used to show things like game results and tuning + event intermediate status. + + A newline is added to the message. + + """ + self.history_logger(s) + + # List of Settings (subclasses can override, and should include these) + global_settings = [ + Setting('description', allow_none(interpret_as_utf8_stripped), + default=None), + ] + + def initialise_from_control_file(self, config): + """Initialise competition data from the control file. + + config -- namespace produced by the control file. + + (When resuming from saved state, this is called before set_state()). + + This processes all global_settings and sets attributes (named by the + setting names). + + It also handles the following settings and sets the corresponding + attributes: + players -- map player code -> game_jobs.Player + + Raises ControlFileError with a description if the control file has a bad + or missing value. + + """ + # This is called for all commands, so it mustn't log anything. + + # Implementations in subclasses should have their own backstop exception + # handlers, so they can at least show what part of the control file was + # being interpreted when the exception occurred. + + # We should accept that there may be unexpected exceptions, because + # control files are allowed to do things like substitute list-like + # objects for Python lists. + + try: + to_set = load_settings(self.global_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + for name, value in to_set.items(): + setattr(self, name, value) + + def interpret_pc(v): + if not isinstance(v, Player_config): + raise ValueError("not a Player") + return v + settings = [ + Setting('players', + interpret_map_of(interpret_identifier, interpret_pc)) + ] + try: + specials = load_settings(settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + self.players = {} + for player_code, player_config in specials['players']: + try: + player = self.game_jobs_player_from_config( + player_code, player_config) + except Exception, e: + raise ControlFileError("player %s: %s" % (player_code, e)) + self.players[player_code] = player + + def game_jobs_player_from_config(self, code, player_config): + """Make a game_jobs.Player from a Player_config. + + Raises ControlFileError with a description if there is an error in the + configuration. + + Returns an incomplete game_jobs.Player (see get_game() for details). + + """ + arguments = player_config.resolve_arguments() + config = load_settings(_player_settings, arguments) + + player = game_jobs.Player() + player.code = code + + try: + player.cmd_args = config['command'] + if '/' in player.cmd_args[0]: + player.cmd_args[0] = self.resolve_pathname(player.cmd_args[0]) + except Exception, e: + raise ControlFileError("'command': %s" % e) + + try: + player.cwd = self.resolve_pathname(config['cwd']) + except Exception, e: + raise ControlFileError("'cwd': %s" % e) + player.environ = config['environ'] + + player.is_reliable_scorer = config['is_reliable_scorer'] + player.allow_claim = config['allow_claim'] + + player.startup_gtp_commands = [] + try: + for v in config['startup_gtp_commands']: + try: + if isinstance(v, basestring): + words = interpret_8bit_string(v).split() + else: + words = list(v) + if not all(gtp_controller.is_well_formed_gtp_word(word) + for word in words): + raise StandardError + except Exception: + raise ValueError("invalid command %s" % v) + player.startup_gtp_commands.append((words[0], words[1:])) + except ValueError, e: + raise ControlFileError("'startup_gtp_commands': %s" % e) + + player.gtp_aliases = {} + try: + for cmd1, cmd2 in config['gtp_aliases']: + if not gtp_controller.is_well_formed_gtp_word(cmd1): + raise ValueError("invalid command %s" % clean_string(cmd1)) + if not gtp_controller.is_well_formed_gtp_word(cmd2): + raise ValueError("invalid command %s" % clean_string(cmd2)) + player.gtp_aliases[cmd1] = cmd2 + except ValueError, e: + raise ControlFileError("'gtp_aliases': %s" % e) + + if config['discard_stderr']: + player.discard_stderr = True + + return player + + + def set_clean_status(self): + """Reset competition state to its initial value.""" + # This is called before logging is set up, so it mustn't log anything. + raise NotImplementedError + + def get_status(self): + """Return full state of the competition, so it can be resumed later. + + The returned result must be pickleable. + + """ + raise NotImplementedError + + def set_status(self, status): + """Reset competition state to a previously reported value. + + 'status' will be a value previously reported by get_status(). + + If the status is invalid, CompetitionError may be raised with a + description of the error, or any other exception may be raised without + a friendly description. + + """ + # This is called for the 'show' command, so it mustn't log anything. + raise NotImplementedError + + def get_player_checks(self): + """List the Player_checks for check_players() to check. + + Returns a list of game_jobs.Player_check objects. The players' + stderr_pathname attribute will be ignored. + + This is called without the competition status being set. + + """ + raise NotImplementedError + + def get_game(self): + """Return the details of the next game to play. + + Returns a game_jobs.Game_job, or NoGameAvailable. + + The following Game_job attributes are left for the ringmaster to set: + - sgf_game_name + - sgf_filename + - sgf_dirname + - void_sgf_dirname + - gtp_log_pathname + - stderr_pathname + + """ + raise NotImplementedError + + def process_game_result(self, response): + """Process the results from a completed game. + + response -- game_jobs.Game_job_result + + This may return a text description of the game result, to override the + default (it should normally include response.game_result.sgf_result). + + It's common for this method to write to the history file. + + """ + raise NotImplementedError + + def process_game_error(self, job, previous_error_count): + """Process a report that a job failed. + + job -- game_jobs.Game_job + previous_error_count -- int >= 0 + + Returns a pair of bools (stop_competition, retry_game) + + If stop_competition is True, the ringmaster will stop starting new + games. Otherwise, if retry_game is true the ringmaster will try running + the same game again. + + The job is one previously returned by get_game(). previous_error_count + is the number of times that this particular job has failed before. + + Failed jobs are ones in which there was an error more serious than one + which just causes an engine to forfeit the game. For example, the job + will fail if one of the engines fails to respond to GTP commands at all, + or (in particular) if it exits as soon as it's invoked because it + doesn't like its command-line options. + + """ + raise NotImplementedError + + def write_screen_report(self, out): + """Write a one-screen summary of current competition status. + + out -- writeable file-like object + + This is supposed to fit comfortably on one screen; it's normally + displayed continuously by the ringmaster. Aim for about 30 lines. + + It should end with a newline, but not have additional blank lines at + the end. + + This should focus on describing incomplete competitions usefully. + + """ + raise NotImplementedError + + def write_short_report(self, out): + """Write a short report of the competition status/results. + + out -- writeable file-like object + + This is used for the ringmaster's 'show' command. + + It should include the competition's description attribute. + + It should end with a newline, but not have additional blank lines at + the end. + + This should be useful for both completed and incomplete competitions. + + """ + raise NotImplementedError + + def write_full_report(self, out): + """Write a detailed report of competition status/results. + + out -- writeable file-like object + + This is used for the ringmaster's 'report' command. + + It should include the competition's description attribute. + + It should end with a newline. + + This should focus on describing completed competitions well. + + """ + raise NotImplementedError + + def get_tournament_results(self): + """Return a Tournament_results object for this competition. + + The competition status must be set before you call this. + + (The returned object is 'live', in that it will see new results as they + come in, but don't rely in this behaviour.) + + Expect this to be implemented for tournaments but not tuning events. + + This won't include results for 'ghost' matchups. + + """ + raise NotImplementedError + + +## Helper functions for settings + +def interpret_board_size(i): + i = interpret_int(i) + if i < 2: + raise ValueError("too small") + if i > 25: + raise ValueError("too large") + return i + +def validate_handicap(handicap, handicap_style, board_size): + """Check whether a handicap is allowed. + + handicap -- int or None + handicap_style -- 'free' or 'fixed' + board_size -- int + + Raises ControlFileError with a description if it isn't. + + """ + if handicap is None: + return True + if handicap < 2: + raise ControlFileError("handicap too small") + if handicap_style == 'fixed': + limit = handicap_layout.max_fixed_handicap_for_board_size(board_size) + else: + limit = handicap_layout.max_free_handicap_for_board_size(board_size) + if handicap > limit: + raise ControlFileError( + "%s handicap out of range for board size %d" % + (handicap_style, board_size)) + + +## Helper functions + +def leading_zero_template(ceiling): + """Return a template suitable for formatting numbers less than 'ceiling'. + + ceiling -- int or None + + Returns a string suitable for Python %-formatting numbers from 0 to + ceiling-1, with leading zeros so that the strings have constant length. + + If ceiling is None, there will be no leading zeros. + + That is, the result is either '%d' or '%0Nd' for some N. + + """ + if ceiling is None: + return "%d" + else: + zeros = len(str(ceiling-1)) + return "%%0%dd" % zeros + + +## Common settings + +game_settings = [ + Setting('board_size', interpret_board_size), + Setting('komi', interpret_float), + Setting('handicap', allow_none(interpret_int), default=None), + Setting('handicap_style', interpret_enum('fixed', 'free'), default='fixed'), + Setting('move_limit', interpret_positive_int, default=1000), + Setting('scorer', interpret_enum('internal', 'players'), default='players'), + Setting('internal_scorer_handicap_compensation', + interpret_enum('no', 'full', 'short'), default='full'), + ] + diff --git a/gomill/gomill/game_jobs.py b/gomill/gomill/game_jobs.py new file mode 100644 index 0000000..537ec00 --- /dev/null +++ b/gomill/gomill/game_jobs.py @@ -0,0 +1,411 @@ +"""Connection between GTP games and the job manager.""" + +import datetime +import os + +from gomill import gtp_controller +from gomill import gtp_games +from gomill import job_manager +from gomill import sgf +from gomill.gtp_controller import BadGtpResponse, GtpChannelError + +class Player(object): + """Player description for Game_jobs. + + required attributes: + code -- short string + cmd_args -- list of strings, as for subprocess.Popen + + optional attributes: + is_reliable_scorer -- bool (default True) + allow_claim -- bool (default False) + gtp_aliases -- map command string -> command string + startup_gtp_commands -- list of pairs (command_name, arguments) + discard_stderr -- bool (default False) + cwd -- working directory to change to (default None) + environ -- maplike of environment variables (default None) + + See gtp_controllers.Gtp_controller for an explanation of gtp_aliases. + + The startup commands will be executed before starting the game. Their + responses will be ignored, but the game will be aborted if any startup + command returns an error. + + By default, the player will be given a copy of the parent process's + environment variables; use 'environ' to add variables or replace particular + values. + + Players are suitable for pickling. + + """ + def __init__(self): + self.is_reliable_scorer = True + self.allow_claim = False + self.gtp_aliases = {} + self.startup_gtp_commands = [] + self.discard_stderr = False + self.cwd = None + self.environ = None + + def make_environ(self): + """Return environment variables to use with the player's subprocess. + + Returns a dict suitable for use with a Subprocess_channel, or None. + + """ + if self.environ is not None: + environ = os.environ.copy() + environ.update(self.environ) + else: + environ = None + return environ + + def copy(self, code): + """Return an independent clone of the Player.""" + result = Player() + result.code = code + result.cmd_args = list(self.cmd_args) + result.is_reliable_scorer = self.is_reliable_scorer + result.allow_claim = self.allow_claim + result.gtp_aliases = dict(self.gtp_aliases) + result.startup_gtp_commands = list(self.startup_gtp_commands) + result.discard_stderr = self.discard_stderr + result.cwd = self.cwd + if self.environ is None: + result.environ = None + else: + result.environ = dict(self.environ) + return result + +class Game_job_result(object): + """Information returned after a worker process plays a game. + + Public attributes: + game_id -- short string + game_data -- arbitrary (copied from the Game_job) + game_result -- gtp_games.Game_result + warnings -- list of strings + log_entries -- list of strings + engine_names -- map player code -> string + engine_descriptions -- map player code -> string + + Game_job_results are suitable for pickling. + + """ + +class Game_job(object): + """A game to be played in a worker process. + + A Game_job is designed to be used a job object for the job manager. That is, + its public interface is the run() method. + + When the job is run, it plays a GTP game as described by its attributes, and + optionally writes an SGF file. The job result is a Game_job_result object. + + required attributes: + game_id -- short string + player_b -- Player + player_w -- Player + board_size -- int + komi -- float + move_limit -- int + + optional attributes (default None unless otherwise stated): + game_data -- arbitrary pickleable data + handicap -- int + handicap_is_free -- bool (default False) + use_internal_scorer -- bool (default True) + internal_scorer_handicap_compensation -- 'no' , 'short', or 'full' + (default 'no') + sgf_filename -- filename for the SGF file + sgf_dirname -- directory pathname for the SGF file + void_sgf_dirname -- directory pathname for the SGF file for void games + sgf_game_name -- string to show as SGF Game Name (default game_id) + sgf_event -- string to show as SGF EVent + sgf_note -- multiline string to put into SGF root comment + gtp_log_pathname -- pathname to use for the GTP log + stderr_pathname -- pathname to send players' stderr to + + The game_id will be returned in the job result, so you can tell which game + you're getting the result for. It also appears in a comment in the SGF file. + + game_data is returned in the job result. It's provided as a convenient way + to pass a small amount of information from get_job() to process_response(). + + If use_internal_scorer is False, the Players' is_reliable_scorer attributes + are used to decide which player is asked to score the game (if both are + marked as reliable, black will be tried before white). + + If sgf_dirname and sgf_filename are set, an SGF file will be written after + the game is over. + + If void_sgf_dirname and sgf_filename are set, an SGF file will be written + for void games (games which were aborted due to unhandled errors). The + leaf directory will be created if necessary. + + If gtp_log_pathname is set, all GTP messages to and from both players will + be logged (this doesn't append; any existing file will be overwritten). + + If stderr_pathname is set, the specified file will be opened in append mode + and both players' standard error streams will be sent there. Otherwise the + players' standard error streams will be left as the standard error of the + calling process. But if a player has discard_stderr=True then its standard + error is sent to os.devnull instead. + + Game_jobs are suitable for pickling. + + """ + def __init__(self): + self.handicap = None + self.handicap_is_free = False + self.sgf_filename = None + self.sgf_dirname = None + self.void_sgf_dirname = None + self.sgf_game_name = None + self.sgf_event = None + self.sgf_note = None + self.use_internal_scorer = True + self.internal_scorer_handicap_compensation = 'no' + self.game_data = None + self.gtp_log_pathname = None + self.stderr_pathname = None + + # The code here has to be happy to run in a separate process. + + def run(self): + """Run the job. + + This method is called by the job manager. + + Returns a Game_job_result, or raises JobFailed. + + """ + self._files_to_close = [] + try: + return self._run() + finally: + # These files are all either flushed after every write, or not + # written to at all from this process, so there shouldn't be any + # errors from close(). + for f in self._files_to_close: + try: + f.close() + except EnvironmentError: + pass + + def _start_player(self, game, colour, player, gtp_log_file): + if player.discard_stderr: + stderr_pathname = os.devnull + else: + stderr_pathname = self.stderr_pathname + if stderr_pathname is not None: + stderr = open(stderr_pathname, "a") + self._files_to_close.append(stderr) + else: + stderr = None + if player.allow_claim: + game.set_claim_allowed(colour) + game.set_player_subprocess( + colour, player.cmd_args, + env=player.make_environ(), cwd=player.cwd, stderr=stderr) + controller = game.get_controller(colour) + controller.set_gtp_aliases(player.gtp_aliases) + if gtp_log_file is not None: + controller.channel.enable_logging( + gtp_log_file, prefix="%s: " % colour) + for command, arguments in player.startup_gtp_commands: + game.send_command(colour, command, *arguments) + + def _run(self): + warnings = [] + log_entries = [] + try: + game = gtp_games.Game(self.board_size, self.komi, self.move_limit) + game.set_player_code('b', self.player_b.code) + game.set_player_code('w', self.player_w.code) + game.set_game_id(self.game_id) + except ValueError, e: + raise job_manager.JobFailed("error creating game: %s" % e) + if self.use_internal_scorer: + game.use_internal_scorer(self.internal_scorer_handicap_compensation) + else: + if self.player_b.is_reliable_scorer: + game.allow_scorer('b') + if self.player_w.is_reliable_scorer: + game.allow_scorer('w') + + if self.gtp_log_pathname is not None: + gtp_log_file = open(self.gtp_log_pathname, "w") + self._files_to_close.append(gtp_log_file) + else: + gtp_log_file = None + + try: + self._start_player(game, 'b', self.player_b, gtp_log_file) + self._start_player(game, 'w', self.player_w, gtp_log_file) + game.request_engine_descriptions() + game.ready() + if self.handicap: + try: + game.set_handicap(self.handicap, self.handicap_is_free) + except ValueError: + raise BadGtpResponse("invalid handicap") + game.run() + except (GtpChannelError, BadGtpResponse), e: + game.close_players() + msg = "aborting game due to error:\n%s" % e + self._record_void_game(game, msg) + late_error_messages = game.describe_late_errors() + if late_error_messages is not None: + msg += "\nalso:\n" + late_error_messages + raise job_manager.JobFailed(msg) + if game.result.is_forfeit: + warnings.append(game.result.detail) + game.close_players() + late_error_messages = game.describe_late_errors() + if late_error_messages: + log_entries.append(late_error_messages) + self._record_game(game) + response = Game_job_result() + response.game_id = self.game_id + response.game_result = game.result + response.warnings = warnings + response.log_entries = log_entries + response.engine_names = game.engine_names + response.engine_descriptions = game.engine_descriptions + response.game_data = self.game_data + return response + + def _write_sgf(self, pathname, sgf_string): + f = open(pathname, "w") + f.write(sgf_string) + f.close() + + def _mkdir(self, pathname): + os.mkdir(pathname) + + def _write_game_record(self, pathname, game, + game_end_message=None, result=None): + b_player = game.players['b'] + w_player = game.players['w'] + notes = [] + sgf_game = game.make_sgf(game_end_message) + root = sgf_game.get_root() + if self.sgf_game_name is not None: + root.set('GN', self.sgf_game_name) + if self.sgf_event is not None: + root.set('EV', self.sgf_event) + notes.append("Event: %s" % self.sgf_event) + notes += [ + "Game id %s" % self.game_id, + "Date %s" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M"), + ] + if game.result is not None: + notes.append("Result %s" % game.result.describe(),) + elif result is not None: + root.set('RE', result) + if self.sgf_note is not None: + notes.append(self.sgf_note) + if game.result is not None: + for player in [b_player, w_player]: + cpu_time = game.result.cpu_times[player] + if cpu_time is not None and cpu_time != "?": + notes.append("%s cpu time: %ss" % + (player, "%.2f" % cpu_time)) + notes += [ + "Black %s %s" % (b_player, game.engine_descriptions[b_player]), + "White %s %s" % (w_player, game.engine_descriptions[w_player]), + ] + root.set('C', "\n".join(notes)) + self._write_sgf(pathname, sgf_game.serialise()) + + def _record_game(self, game): + """Record the game in the standard sgf directory.""" + if self.sgf_dirname is None or self.sgf_filename is None: + return + pathname = os.path.join(self.sgf_dirname, self.sgf_filename) + self._write_game_record(pathname, game) + + def _record_void_game(self, game, game_end_message): + """Record the game in the void sgf directory if it had any moves.""" + if not game.moves: + return + if self.void_sgf_dirname is None or self.sgf_filename is None: + return + if not os.path.exists(self.void_sgf_dirname): + self._mkdir(self.void_sgf_dirname) + pathname = os.path.join(self.void_sgf_dirname, self.sgf_filename) + self._write_game_record(pathname, game, game_end_message, result='Void') + + +class CheckFailed(StandardError): + """Error reported by check_player()""" + +class Player_check(object): + """Information required to check a player. + + required attributes: + player -- Player + board_size -- int + komi -- float + + """ + +def check_player(player_check, discard_stderr=False): + """Do a test run of a GTP engine. + + player_check -- Player_check object + + This starts an engine subprocess, sends it some GTP commands, and ends the + process again. + + Raises CheckFailed if the player doesn't pass the checks. + + Returns a list of warning messages. + + Currently checks: + - any explicitly specified cwd exists and is a directory + - the engine subprocess starts, and replies to GTP commands + - the engine reports protocol version 2 (if it supports protocol_version) + - the engine accepts any startup_gtp_commands + - the engine accepts the specified board size and komi + - the engine accepts the 'clear_board' command + - the engine accepts 'quit' and closes down cleanly + + """ + player = player_check.player + if player.cwd is not None and not os.path.isdir(player.cwd): + raise CheckFailed("bad working directory: %s" % player.cwd) + + if discard_stderr: + stderr = open(os.devnull, "w") + else: + stderr = None + try: + try: + channel = gtp_controller.Subprocess_gtp_channel( + player.cmd_args, + env=player.make_environ(), cwd=player.cwd, stderr=stderr) + except GtpChannelError, e: + raise GtpChannelError( + "error starting subprocess for %s:\n%s" % (player.code, e)) + controller = gtp_controller.Gtp_controller(channel, player.code) + controller.set_gtp_aliases(player.gtp_aliases) + controller.check_protocol_version() + for command, arguments in player.startup_gtp_commands: + controller.do_command(command, *arguments) + controller.do_command("boardsize", str(player_check.board_size)) + controller.do_command("clear_board") + controller.do_command("komi", str(player_check.komi)) + controller.safe_close() + except (GtpChannelError, BadGtpResponse), e: + raise CheckFailed(str(e)) + else: + return controller.retrieve_error_messages() + finally: + try: + if stderr is not None: + stderr.close() + except Exception: + pass + diff --git a/gomill/gomill/gtp_controller.py b/gomill/gomill/gtp_controller.py new file mode 100644 index 0000000..f0bb128 --- /dev/null +++ b/gomill/gomill/gtp_controller.py @@ -0,0 +1,823 @@ +"""Go Text Protocol support (controller side). + +Based on GTP 'draft version 2' (see ). + +""" + +import errno +import os +import re +import signal +import subprocess + +from gomill.utils import * +from gomill.common import * + + +class GtpChannelError(StandardError): + """Low-level error trying to talk to a GTP engine. + + This is the base class for GtpProtocolError, GtpTransportError, + and GtpChannelClosed. It may also be raised directly. + + """ + +class GtpProtocolError(GtpChannelError): + """A GTP engine returned an ill-formed response.""" + +class GtpTransportError(GtpChannelError): + """An error from the transport underlying the GTP channel.""" + +class GtpChannelClosed(GtpChannelError): + """The (command or response) channel to a GTP engine has been closed.""" + + +class BadGtpResponse(StandardError): + """Unacceptable response from a GTP engine. + + This is usually used to indicate a GTP failure ('?') response. + + Some higher-level functions use this exception to indicate a GTP success + ('=') response which they couldn't interpret. + + Additional attributes: + gtp_command -- string (or None) + gtp_arguments -- sequence of strings (or None) + gtp_error_message -- string (or None) + + """ + def __init__(self, args, + gtp_command=None, gtp_arguments=None, gtp_error_message=None): + StandardError.__init__(self, args) + self.gtp_command = gtp_command + self.gtp_arguments = gtp_arguments + self.gtp_error_message = gtp_error_message + + +_gtp_word_characters_re = re.compile(r"\A[\x21-\x7e\x80-\xff]+\Z") +_remove_response_controls_re = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") + +def is_well_formed_gtp_word(s): + """Check whether 's' is well-formed as a single GTP word. + + In particular, this rejects unicode objects and strings containing spaces. + + """ + if not isinstance(s, str): + return False + if not _gtp_word_characters_re.search(s): + return False + return True + +class Gtp_channel(object): + """A communication channel to a GTP engine. + + public attributes: + exit_status + resource_usage + + exit_status describes the engine's exit status as an integer. It is None if + not available. The integer is in the form returned by os.wait() (in + particular, zero for successful exit, nonzero for unsuccessful). + + resource_usage describes the engine's resource usage (see + resource.getrusage() for the format). It is None if not available. + + In practice these attributes are only available for subprocess-based + channels, and only after they've been closed. + + """ + def __init__(self): + self.exit_status = None + self.resource_usage = None + self.log_dest = None + self.log_prefix = None + + def enable_logging(self, log_dest, prefix=""): + """Log all messages sent and received over the channel. + + log_dest -- writable file-like object (eg an open file) + prefix -- short string to prepend to logged lines + + """ + self.log_dest = log_dest + self.log_prefix = prefix + + def _log(self, marker, message): + """Log a message. + + marker -- string that goes before the log prefix + message -- string to log + + Swallows all errors. + + """ + try: + self.log_dest.write(marker + self.log_prefix + message + "\n") + self.log_dest.flush() + except Exception: + pass + + def send_command(self, command, arguments): + """Send a GTP command over the channel. + + command -- string + arguments -- list of strings + + May raise GtpChannelError. + + Raises ValueError if the command or an argument contains a character + forbidden in GTP. + + """ + if not is_well_formed_gtp_word(command): + raise ValueError("bad command") + for argument in arguments: + if not is_well_formed_gtp_word(argument): + raise ValueError("bad argument") + if self.log_dest is not None: + self._log(">> ", command + ("".join(" " + a for a in arguments))) + self.send_command_impl(command, arguments) + + def get_response(self): + """Read a GTP response from the channel. + + Waits indefinitely for the response. + + Returns a pair (is_failure, response) + + 'is_failure' is a bool indicating whether the engine returned a success + or a failure response. + + For a success response, 'response' is the result from the engine; for a + failure response it's the error message from the engine. + + This cleans the response according to the GTP spec, and also removes + leading and trailing whitespace. + + This means that 'response' is an 8-bit string with no trailing + whitespace. It may contain newlines, but there are no empty lines except + perhaps the first. There is no leading whitespace on the first line. + There are no other control characters. It may include 'high' characters, + in whatever encoding the engine was using. + + May raise GtpChannelError. In particular, raises GtpProtocolError if the + success/failure indicator can't be read from the engine's response. + + """ + result = self.get_response_impl() + if self.log_dest is not None: + is_error, response = result + if is_error: + response = "? " + response + else: + response = "= " + response + self._log("<< ", response.rstrip()) + return result + + # For subclasses to override: + + def close(self): + """Close the command and response channels. + + Channel implementations may use this to clean up resources associated + with the engine (eg, to terminate a subprocess). + + Raises GtpTransportError if a serious error is detected while doing this + (this is unlikely in practice). + + When it is meaningful (eg, for subprocess channels) this waits for the + engine to exit. Nonzero exit status is not considered a serious error. + + """ + pass + + def send_command_impl(self, command, arguments): + raise NotImplementedError + + def get_response_impl(self): + raise NotImplementedError + + +class Internal_gtp_channel(Gtp_channel): + """A GTP channel connected to an in-process Python GTP engine. + + Instantiate with a Gtp_engine_protocol object. + + This waits to invoke the engine's handler for each command until the + correponding response is requested. + + """ + def __init__(self, engine): + Gtp_channel.__init__(self) + self.engine = engine + self.outstanding_commands = [] + self.session_is_ended = False + + def send_command_impl(self, command, arguments): + if self.session_is_ended: + raise GtpChannelClosed("engine has ended the session") + self.outstanding_commands.append((command, arguments)) + + def get_response_impl(self): + if self.session_is_ended: + raise GtpChannelClosed("engine has ended the session") + try: + command, arguments = self.outstanding_commands.pop(0) + except IndexError: + raise GtpChannelError("no outstanding commands") + is_error, response, end_session = \ + self.engine.run_command(command, arguments) + if end_session: + self.session_is_ended = True + return is_error, response + + +class Linebased_gtp_channel(Gtp_channel): + """Generic Gtp_channel based on line-by-line communication.""" + + def __init__(self): + Gtp_channel.__init__(self) + self.is_first_response = True + + # Not using command ids; I don't see the need unless we see problems in + # practice with engines getting out of sync. + + def send_command_impl(self, command, arguments): + words = [command] + arguments + self.send_command_line(" ".join(words) + "\n") + + def get_response_impl(self): + """Obtain response according to GTP protocol. + + If we receive EOF before any data, we raise GtpChannelClosed. + + If we receive EOF otherwise, we use the data received anyway. + + The first time this is called, we check the first byte without reading + the whole line, and raise GtpProtocolError if it isn't plausibly the + start of a GTP response (strictly, if it's a control character we should + just discard it, but I think it's more useful to reject them here; in + particular, this lets us detect GMP). + + """ + lines = [] + seen_data = False + peeked_byte = None + if self.is_first_response: + self.is_first_response = False + # We read one byte first so that we don't hang if the engine never + # sends a newline (eg, it's speaking GMP). + try: + peeked_byte = self.get_response_byte() + except NotImplementedError: + pass + else: + if peeked_byte == "": + raise GtpChannelClosed( + "engine has closed the response channel") + if peeked_byte == "\x01": + raise GtpProtocolError( + "engine appears to be speaking GMP, not GTP!") + # These are the characters which could legitimately start a GTP + # response. In principle, we should be discarding other controls + # rather than treating them as errors, but it's more useful to + # report a protocol error. + if peeked_byte not in (' ', '\t', '\r', '\n', '#', '=', '?'): + raise GtpProtocolError( + "engine isn't speaking GTP: " + "first byte is %s" % repr(peeked_byte)) + if peeked_byte == "\n": + peeked_byte = None + while True: + s = self.get_response_line() + if peeked_byte: + s = peeked_byte + s + peeked_byte = None + # << All other [than HT, CR, LF] control characters must be + # discarded on input >> + # << Any occurence of a CR character must be discarded on input >> + s = _remove_response_controls_re.sub("", s) + # << Empty lines and lines with only whitespace sent by the engine + # and occuring outside a response must be ignored by the + # controller >> + if not seen_data: + if s.strip() == "": + if s.endswith("\n"): + continue + else: + break + else: + seen_data = True + if s == "\n": + break + lines.append(s) + if not s.endswith("\n"): + break + if not lines: + # Means 'EOF and empty response' + raise GtpChannelClosed("engine has closed the response channel") + first_line = lines[0] + # It's certain that first line isn't empty + if first_line[0] == "?": + is_error = True + elif first_line[0] == "=": + is_error = False + else: + raise GtpProtocolError( + "no success/failure indication from engine: " + "first line is `%s`" % first_line.rstrip()) + lines[0] = first_line[1:].lstrip(" \t") + response = "".join(lines).rstrip() + response = response.replace("\t", " ") + return is_error, response + + + # For subclasses to override: + + def send_command_line(self, command): + """Send a line of text over the channel. + + command -- string terminated by a newline. + + May raise GtpChannelClosed or GtpTransportError + + """ + raise NotImplementedError + + def get_response_line(self): + """Read a line of text from the channel. + + May raise GtpTransportError + + The result ends in a newline unless end-of-file was seen (ie, the same + protocol to indicate end-of-file as Python's readline()). + + This blocks until a line is available, or end-of-file is reached. + + """ + raise NotImplementedError + + def get_response_byte(self): + """Read a single byte from the channel. + + May raise GtpTransportError + + This blocks until a byte is available, or end-of-file is reached. + + Subclasses don't have to implement this. + + """ + raise NotImplementedError + + +def permit_sigpipe(): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +class Subprocess_gtp_channel(Linebased_gtp_channel): + """A GTP channel to a subprocess. + + Instantiate with + command -- list of strings (as for subprocess.Popen) + stderr -- destination for standard error output (optional) + cwd -- working directory to change to (optional) + env -- new environment (optional) + Instantiation will raise GtpChannelError if the process can't be started. + + This starts the subprocess and speaks GTP over its standard input and + output. + + By default, the subprocess's standard error is left as the standard error of + the calling process. The 'stderr' parameter is interpreted as for + subprocess.Popen (but don't set it to STDOUT or PIPE). + + The 'cwd' and 'env' parameters are interpreted as for subprocess.Popen. + + Closing the channel waits for the subprocess to exit. + + """ + def __init__(self, command, stderr=None, cwd=None, env=None): + Linebased_gtp_channel.__init__(self) + try: + p = subprocess.Popen( + command, + preexec_fn=permit_sigpipe, close_fds=True, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=stderr, cwd=cwd, env=env) + except EnvironmentError, e: + raise GtpChannelError(str(e)) + self.subprocess = p + self.command_pipe = p.stdin + self.response_pipe = p.stdout + + def send_command_line(self, command): + try: + self.command_pipe.write(command) + self.command_pipe.flush() + except EnvironmentError, e: + if e.errno == errno.EPIPE: + raise GtpChannelClosed("engine has closed the command channel") + else: + raise GtpTransportError(str(e)) + + def get_response_line(self): + try: + return self.response_pipe.readline() + except EnvironmentError, e: + raise GtpTransportError(str(e)) + + def get_response_byte(self): + try: + return self.response_pipe.read(1) + except EnvironmentError, e: + raise GtpTransportError(str(e)) + + def close(self): + # Errors from closing pipes or wait4() are unlikely, but possible. + + # Ideally would give up waiting after a while and forcibly terminate the + # subprocess. + errors = [] + try: + self.command_pipe.close() + except EnvironmentError, e: + errors.append("error closing command pipe:\n%s" % e) + try: + self.response_pipe.close() + except EnvironmentError, e: + errors.append("error closing response pipe:\n%s" % e) + errors.append(str(e)) + try: + # We don't really care about the exit status, but we do want to be + # sure it isn't still running. + # Even if there were errors closing the pipes, it's most likely that + # the subprocesses has exited. + pid, exit_status, rusage = os.wait4(self.subprocess.pid, 0) + self.exit_status = exit_status + self.resource_usage = rusage + except EnvironmentError, e: + errors.append(str(e)) + if errors: + raise GtpTransportError("\n".join(errors)) + + +class Gtp_controller(object): + """Implementation of the controller side of the GTP protocol. + + This communicates with a single engine. It's a higher level interface than + Gtp_channel, including helper functions for the protocol-level GTP commands. + + Public attributes: + channel -- the underlying Gtp_channel + name -- the channel name (used in error messages) + channel_is_closed -- bool + channel_is_bad -- bool + + It's ok to access the underlying channel directly (eg, to enable logging). + + Instantiate with channel and name. + + """ + def __init__(self, channel, name): + self.channel = channel + self.name = str(name) + self.known_commands = {} + self.log_dest = None + self.gtp_aliases = {} + self.is_first_command = True + self.errors_seen = [] + self.channel_is_closed = False + self.channel_is_bad = False + + def do_command(self, command, *arguments): + """Send a command to the engine and return the response. + + command -- string (command name) + arguments -- strings or unicode objects + + Arguments may not contain spaces. If a command is documented as + expecting a list of vertices, each vertex must be passed as a separate + argument. + + Arguments may be unicode objects, in which case they will be sent as + utf-8. + + + Returns the result text from the engine as an 8-bit string with no + trailing whitespace. It may contain newlines, but there are no empty + lines except perhaps the first. There is no leading whitespace on the + first line. There are no other control characters. It may include 'high' + characters, in whatever encoding the engine was using. (The result text + doesn't include the leading =[id] bit.) + + If the engine returns a failure response, raises BadGtpResponse (use the + gtp_error_message attribute to retrieve the text of the response). + + This will wait indefinitely for the engine to produce the response. + + + Raises GtpChannelClosed if the engine has apparently closed its + connection. + + Raises GtpProtocolError if the engine's response is too mangled to be + returned. + + Raises GtpTransportError if there was an error from the communication + layer between the controller and the engine (which may well mean that + the engine has gone away). + + If any of these GtpChannelError variants is raised, this also marks the + channel as 'bad' (this has no effect on future do_command() calls, but + see safe_do_command() below). + + + This applies gtp_aliases (see below). Error messages (including + BadGtpResponse.gtp_command) will refer to the underlying command, not + the alias. + + """ + if self.channel_is_closed: + raise StandardError("channel is closed") + + def fix_argument(argument): + if isinstance(argument, unicode): + return argument.encode("utf-8") + else: + return argument + + fixed_command = fix_argument(command) + fixed_arguments = map(fix_argument, arguments) + translated_command = self.gtp_aliases.get(fixed_command, fixed_command) + is_first_command = self.is_first_command + self.is_first_command = False + + def format_command(): + desc = "%s" % (" ".join([translated_command] + fixed_arguments)) + if is_first_command: + return "first command (%s)" % desc + else: + return "'%s'" % desc + + try: + is_sending = True + self.channel.send_command(translated_command, fixed_arguments) + is_sending = False + is_failure, response = self.channel.get_response() + except GtpChannelError, e: + self.channel_is_bad = True + if isinstance(e, GtpTransportError): + error_label = "transport error" + elif isinstance(e, GtpProtocolError): + error_label = "GTP protocol error" + else: + error_label = "error" + if is_sending: + msg = "%s sending %s to %s:\n%s" + else: + msg = "%s reading response to %s from %s:\n%s" + e.args = (msg % (error_label, format_command(), self.name, e),) + raise + if is_failure: + raise BadGtpResponse( + "failure response from %s to %s:\n%s" % + (format_command(), self.name, response), + gtp_command=translated_command, gtp_arguments=fixed_arguments, + gtp_error_message=response) + return response + + def _known_command(self, command, do_command): + """Common implementation for known_command and safe_known_command.""" + result = self.known_commands.get(command) + if result is not None: + return result + translated_command = self.gtp_aliases.get(command, command) + try: + response = do_command("known_command", translated_command) + except BadGtpResponse: + known = False + else: + known = (response == 'true') + self.known_commands[command] = known + return known + + def known_command(self, command): + """Check whether 'command' is known by the engine. + + This sends 'known_command' the first time it's asked, then caches the + result. + + If known_command fails, returns False. + + May propagate GtpChannelError (see do_command). + + This does the right thing if gtp aliases have been set (but it doesn't + invalidate the cache if they're changed). + + """ + return self._known_command(command, self.do_command) + + def check_protocol_version(self): + """Check the engine's declared protocol version. + + Raises BadGtpResponse if the engine declares a version other than 2. + Otherwise does nothing. + + If the engine returns a GTP failure response (in particular, if + protocol_version isn't implemented), this does nothing. + + May propagate GtpChannelError (see do_command). + + """ + try: + protocol_version = self.do_command("protocol_version") + except BadGtpResponse: + return + if protocol_version != "2": + raise BadGtpResponse( + "%s reports GTP protocol version %s" % + (self.name, protocol_version)) + + def list_commands(self): + """Return the engine's declared command list. + + Returns a list of nonempty strings without leading or trailing + whitespace. Filters out strings which wouldn't be accepted as commands. + + May propagate GtpChannelError or BadGtpResponse + + """ + response = self.do_command('list_commands') + stripped = [s for s in + (t.strip() for t in response.split("\n"))] + return [s for s in stripped if is_well_formed_gtp_word(s)] + + def close(self): + """Close the communication channel to the engine. + + May propagate GtpTransportError. + + Unless you have a good reason, you should send 'quit' before closing the + connection (eg, by using safe_close() instead of close()). + + When it is meaningful (eg, for subprocess channels) this waits for the + engine to exit. Nonzero exit status is not considered an error. + + """ + if self.channel_is_closed: + raise StandardError("channel is closed") + try: + self.channel.close() + except GtpTransportError, e: + raise GtpTransportError( + "error closing %s:\n%s" % (self.name, e)) + self.channel_is_closed = True + + def safe_do_command(self, command, *arguments): + """Variant of do_command which sets low-level exceptions aside. + + If the channel is closed or marked bad, this does not attempt to send + the command, and returns None. + + If GtpChannelError is raised while running the command, it is not + propagated, but the error message is recorded; use + retrieve_error_messages to retrieve these. In this case the function + returns None. + + BadGtpResponse is raised in the same way as for do_command. + + """ + if self.channel_is_bad or self.channel_is_closed: + return None + try: + return self.do_command(command, *arguments) + except BadGtpResponse, e: + raise + except GtpChannelError, e: + self.errors_seen.append(str(e)) + return None + + def safe_known_command(self, command): + """Variant of known_command which sets low-level exceptions aside. + + If result is already cached, returns it. + + Otherwise, if the channel is closed or marked bad, returns False. + + Otherwise acts like known_command above, using safe_do_command to send + the command to the engine. + + """ + return self._known_command(command, self.safe_do_command) + + def safe_close(self): + """Close the communication channel to the engine, avoiding exceptions. + + This is safe to call even if the channel is already closed, or has had + protocol or transport errors. + + This will not propagate any exceptions; it will set them aside like + safe_do_command. + + When it is meaningful (eg, for subprocess channels) this waits for the + engine to exit. Nonzero exit status is not reported as an error. + + + This will send 'quit' to the engine if the channel is not marked as bad. + Any failure response will be set aside. + + """ + if self.channel_is_closed: + return + if not self.channel_is_bad: + try: + self.safe_do_command("quit") + except BadGtpResponse, e: + self.errors_seen.append(str(e)) + try: + self.channel.close() + except GtpTransportError, e: + self.errors_seen.append("error closing %s:\n%s" % (self.name, e)) + self.channel_is_closed = True + + def retrieve_error_messages(self): + """Return error messages which have been set aside by 'safe' commands. + + Returns a list of strings (empty if there are no such messages). + + """ + return self.errors_seen[:] + + + def set_gtp_aliases(self, aliases): + """Set GTP command aliases. + + aliases -- map public command name -> underlying command name + + In future calls to do_command, a request to send 'public command name' + will be sent to the underlying channel as the corresponding 'underlying + command name'. + + """ + self.gtp_aliases = aliases + + +def _fix_version(name, version): + """Clean up version strings.""" + version = sanitise_utf8(version) + if version.lower().startswith(name.lower()): + version = version[len(name):].lstrip() + # Some engines unfortunately include usage instructions in the version + # string (apparently for the sake of kgsGTP); try to clean this up. + if len(version) > 64: + # MoGo + a, b, c = version.partition(". Please read http:") + if b: + return a + # Pachi + a, b, c = version.partition(": I'm playing") + if b: + return a + # Other + return version.split()[0] + return version + +def describe_engine(controller, default="unknown"): + """Retrieve a description of a controller's engine via GTP. + + default -- text to use for the description if all GTP commands fail. + + This uses the 'name', 'version', and 'gomill-describe_engine' commands. + + Returns a pair of utf-8 strings (short, long): + short -- single-line form (engine name, and version if it's not too long) + long -- multi-line form (engine name, version, description) + + Attempts to clean up over-long version strings. + + May propagate GtpChannelError. + + """ + try: + name = sanitise_utf8(controller.do_command("name")) + except BadGtpResponse: + name = default + try: + version = _fix_version(name, controller.do_command("version")) + if version: + if len(version) <= 32: + short_s = name + ":" + version + else: + short_s = name + long_s = name + ":" + version + else: + long_s = short_s = name + except BadGtpResponse: + long_s = short_s = name + + if controller.known_command("gomill-describe_engine"): + try: + long_s = sanitise_utf8( + controller.do_command("gomill-describe_engine")) + except BadGtpResponse: + pass + return short_s, long_s diff --git a/gomill/gomill/gtp_engine.py b/gomill/gomill/gtp_engine.py new file mode 100644 index 0000000..7f73e11 --- /dev/null +++ b/gomill/gomill/gtp_engine.py @@ -0,0 +1,532 @@ +"""Go Text Protocol support (engine side). + +Based on GTP 'draft version 2' (see ), +and gnugo 3.7 as 'reference implementation'. + +""" + +import errno +import re +import sys +import os + +from gomill.common import * +from gomill.utils import isinf, isnan +from gomill import compact_tracebacks + + +class GtpError(StandardError): + """Error reported by a command handler.""" + +class GtpFatalError(GtpError): + """Fatal error reported by a command handler.""" + +class GtpQuit(Exception): + """Request to end session from a command handler.""" + + + + +### Handler support + +def interpret_boolean(arg): + """Interpret a string representing a boolean, as specified by GTP. + + Returns a Python bool. + + Raises GtpError with an appropriate message if 'arg' isn't a valid GTP + boolean specification. + + """ + try: + return {'true': True, 'false': False}[arg] + except KeyError: + raise GtpError("invalid boolean: '%s'" % arg) + +def interpret_colour(arg): + """Interpret a string representing a colour, as specified by GTP. + + Returns 'b' or 'w'. + + Raises GtpError with an appropriate message if 'arg' isn't a valid GTP + colour specification. + + """ + try: + return {'w': 'w', 'white': 'w', 'b': 'b', 'black': 'b'}[arg.lower()] + except KeyError: + raise GtpError("invalid colour: '%s'" % arg) + +def interpret_vertex(arg, board_size): + """Interpret a string representing a vertex, as specified by GTP. + + Returns a pair of coordinates (row, col) in range(0, board_size), + or None for a pass. + + Raises GtpError with an appropriate message if 'arg' isn't a valid GTP + vertex specification for a board of size 'board_size'. + + """ + try: + return move_from_vertex(arg, board_size) + except ValueError, e: + raise GtpError(str(e)) + + +_gtp_int_max = 2**31-1 + +def interpret_int(arg): + """Interpret a string representing an int, as specified by GTP. + + Returns a Python int. + + Raises GtpError with an appropriate message if 'arg' isn't a valid GTP + int specification. + + Negative numbers are returned as -1. Numbers above 2**31-1 are returned as + 2**31-1. + + """ + # I can't tell how gnugo treats negative numbers, except that it counts them + # as integers not in a suitable range for boardsize. The clipping of high + # integers is what it does for command ids. + try: + result = int(arg, 10) + except ValueError: + raise GtpError("invalid int: '%s'" % arg) + if result < 0: + result = -1 + elif result > _gtp_int_max: + result = _gtp_int_max + return result + +def interpret_float(arg): + """Interpret a string representing a float, as specified by GTP. + + Returns a Python float. + + Raises GtpError with an appropriate message if 'arg' isn't a valid GTP + float specification. + + Accepts strings accepted as a float by the platform libc; rejects + infinities and NaNs. + + """ + try: + result = float(arg) + if isinf(result) or isnan(result): + raise ValueError + except ValueError: + raise GtpError("invalid float: '%s'" % arg) + return result + +def format_gtp_boolean(b): + """Format a Python bool in GTP format.""" + if b: + return "true" + else: + return "false" + +def report_bad_arguments(): + """Raise GtpError with a suitable message for invalid arguments. + + Note that gnugo (3.7) seems to ignore extra arguments in practice; it's + supposed to be the reference implementation, so perhaps you should do the + same. + + """ + raise GtpError("invalid arguments") + + + +### Parsing + +_remove_controls_re = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]") +_remove_response_controls_re = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") +_normalise_whitespace_re = re.compile(r"[\x09\x20]+") +_command_id_re = re.compile(r"^-?[0-9]+") + +def _preprocess_line(s): + """Clean up an input line and normalise whitespace.""" + s = s.partition("#")[0] + s = _remove_controls_re.sub("", s) + s = _normalise_whitespace_re.sub(" ", s) + return s + +def _clean_response(response): + """Clean up a proposed response.""" + if response is None: + return "" + if isinstance(response, unicode): + s = response.encode("utf-8") + else: + s = str(response) + s = s.rstrip() + s = s.replace("\n\n", "\n.\n") + s = _remove_response_controls_re.sub("", s) + s = s.replace("\t", " ") + return s + +def _parse_line(line): + """Parse a nonempty input line. + + Returns a tuple (command_id, command, arguments) + command_id -- string + command -- string + arguments -- list of strings + + Returns command None if the line is to be treated as empty after all. + + Behaviour in error cases is copied from gnugo 3.7. + + """ + tokens = line.split() + s = tokens[0] + command_id = None + id_match = _command_id_re.match(s) + if id_match: + command = s[id_match.end():] + if command == "": + try: + command = tokens[1] + except IndexError: + command = None + args = tokens[2:] + else: + args = tokens[1:] + command_id = id_match.group() + int_command_id = int(command_id) + if int_command_id < 0: + command_id = None + elif int_command_id > _gtp_int_max: + command_id = str(_gtp_int_max) + else: + command_id = None + command = s + args = tokens[1:] + return command_id, command, args + + +class Gtp_engine_protocol(object): + """Implementation of the engine side of the GTP protocol. + + Sample use: + e = Gtp_engine_protocol() + e.add_protocol_commands() + e.add_command('foo', foo_handler) + response, end_session = e.handle_line('foo w d5') + + + GTP commands are dispatched to _handler functions_. These can by any Python + callable. The handler function is passed a single parameter, which is a list + of strings representing the command's arguments (nonempty strings of + printable non-whitespace characters). + + The handler should return the response to sent to the controller. You can + use either None or the empty string for an empty response. If the returned + value isn't suitable to be used directly as a GTP response, it will be + 'cleaned up' so that it can be. Unicode objects will be encoded as utf-8. + + To return a failure response, the handler should raise GtpError with an + appropriate message. + + To end the session, the handler should raise GtpQuit or GtpFatalError. Any + exception message will be reported, as a success or failure response + respectively. + + If a handler raises another exception (instance of Exception), this will be + reported as 'internal error', followed by the exception description and + traceback. By default, this is not treated as a fatal error; use + set_handler_exceptions_fatal() to change this. + + """ + + def __init__(self): + self.handlers = {} + self.handler_exceptions_are_fatal = False + + def set_handler_exceptions_fatal(self, b=True): + """Treat exceptions from handlers as fatal errors.""" + self.handler_exceptions_are_fatal = bool(b) + + def add_command(self, command, handler): + """Register the handler function for a command.""" + self.handlers[command] = handler + + def add_commands(self, handlers): + """Register multiple handler functions. + + handlers -- dict command name -> handler + + """ + self.handlers.update(handlers) + + def remove_command(self, command): + """Remove a registered handler function. + + Silently does nothing if no handler was registered for the command. + + """ + try: + del self.handlers[command] + except KeyError: + pass + + def list_commands(self): + """Return a list of known commands.""" + return sorted(self.handlers) + + def _do_command(self, command, args): + try: + handler = self.handlers[command] + except KeyError: + raise GtpError("unknown command") + try: + return handler(args) + except (GtpError, GtpQuit): + raise + except Exception: + traceback = compact_tracebacks.format_traceback(skip=1) + if self.handler_exceptions_are_fatal: + raise GtpFatalError("internal error; exiting\n" + traceback) + else: + raise GtpError("internal error\n" + traceback) + + def run_command(self, command, args): + """Run the handler for a command directly. + + You can use this from Python code to interact with a GTP engine without + going via the GTP line-based syntax. + + command -- string (command name) + arguments -- list of strings (or None) + + Returns a tuple (is_error, response, end_session) + + is_error -- bool + response -- the GTP response + end_session -- bool + + The response is a string, not ending with a newline (or any other + whitespace). + + If end_session is true, the engine doesn't want to receive any more + commands. + + """ + try: + response = self._do_command(command, args) + except GtpQuit, e: + is_error = False + response = e + end_session = True + except GtpFatalError, e: + is_error = True + response = str(e) + if response == "": + response = "unspecified fatal error" + end_session = True + except GtpError, e: + is_error = True + response = str(e) + if response == "": + response = "unspecified error" + end_session = False + else: + is_error = False + end_session = False + return is_error, _clean_response(response), end_session + + def handle_line(self, line): + """Handle a line of input. + + line -- 8-bit string containing one line of input. + + The line may or may not contain the terminating newline. Any internal + newline is discarded. + + Returns a pair (response, end_session) + + response -- the GTP response to be sent to the controller + end_session -- bool + + response is normally a string containing a well-formed GTP response + (ending with '\n\n'). It may also be None, in which case nothing at all + should be sent to the controller. + + If end_session is true, the GTP session should be terminated. + + """ + normalised = _preprocess_line(line) + if normalised == "" or normalised == " ": + return None, False + command_id, command, args = _parse_line(normalised) + if command is None: + # Line with only a command id + return None, False + is_error, cleaned_response, end_session = \ + self.run_command(command, args) + if is_error: + response_code = "?" + else: + response_code = "=" + if command_id is not None: + response_prefix = response_code + command_id + else: + response_prefix = response_code + if cleaned_response == "": + response_sep = "" + else: + response_sep = " " + response = "%s%s%s\n\n" % ( + response_prefix, response_sep, cleaned_response) + return response, end_session + + def handle_known_command(self, args): + # Imitating gnugo's behaviour for bad args + try: + result = (args[0] in self.handlers) + except IndexError: + result = False + return format_gtp_boolean(result) + + def handle_list_commands(self, args): + # Gnugo ignores any arguments + return "\n".join(self.list_commands()) + + def handle_protocol_version(self, args): + # Gnugo ignores any arguments + return "2" + + def handle_quit(self, args): + # Gnugo ignores any arguments + raise GtpQuit + + def add_protocol_commands(self): + """Add the standard protocol-level commands. + + These are the commands which can be handled without reference to the + underlying engine: + known_command + list_commands + protocol_version + quit + + """ + self.add_command("known_command", self.handle_known_command) + self.add_command("list_commands", self.handle_list_commands) + self.add_command("protocol_version", self.handle_protocol_version) + self.add_command("quit", self.handle_quit) + + + +### Session loop + +class ControllerDisconnected(IOError): + """The GTP controller went away.""" + +def _run_gtp_session(engine, read, write): + while True: + try: + line = read() + except EOFError: + break + response, end_session = engine.handle_line(line) + if response is not None: + try: + write(response) + except IOError, e: + if e.errno == errno.EPIPE: + raise ControllerDisconnected(*e.args) + else: + raise + if end_session: + break + +def run_gtp_session(engine, src, dst): + """Run a GTP engine session using 'src' and 'dst' for the controller. + + engine -- Gtp_engine_protocol object + src -- readable file-like object + dst -- writeable file-like object + + Returns either when EOF is seen on src, or when the engine signals end of + session. + + If a write fails with 'broken pipe', this raises ControllerDisconnected. + + """ + def read(): + line = src.readline() + if line == "": + raise EOFError + return line + def write(s): + dst.write(s) + dst.flush() + _run_gtp_session(engine, read, write) + +def make_readline_completer(engine): + """Return a readline completer function for the specified engine.""" + commands = engine.list_commands() + def completer(text, state): + matches = [s for s in commands if s.startswith(text)] + try: + return matches[state] + " " + except IndexError: + return None + return completer + +def run_interactive_gtp_session(engine): + """Run a GTP engine session on stdin and stdout, using readline. + + engine -- Gtp_engine_protocol object + + This enables readline tab-expansion, and command history in + ~/.gomill-gtp-history (if readline is available). + + Returns either when EOF is seen on stdin, or when the engine signals end of + session. + + If stdin isn't a terminal, this is equivalent to run_gtp_session. + + If a write fails with 'broken pipe', this raises ControllerDisconnected. + + Note that this will propagate KeyboardInterrupt if the user presses ^C; + normally you'll want to handle this to avoid an ugly traceback. + + """ + # readline doesn't do anything if stdin isn't a tty, but it's simplest to + # just not import it in that case. + try: + use_readline = os.isatty(sys.stdin.fileno()) + if use_readline: + import readline + except Exception: + use_readline = False + if not use_readline: + run_gtp_session(engine, sys.stdin, sys.stdout) + return + + def write(s): + sys.stdout.write(s) + sys.stdout.flush() + + history_pathname = os.path.expanduser("~/.gomill-gtp-history") + readline.parse_and_bind("tab: complete") + old_completer = readline.get_completer() + old_delims = readline.get_completer_delims() + readline.set_completer(make_readline_completer(engine)) + readline.set_completer_delims("") + try: + readline.read_history_file(history_pathname) + except EnvironmentError: + pass + _run_gtp_session(engine, raw_input, write) + try: + readline.write_history_file(history_pathname) + except EnvironmentError: + pass + readline.set_completer(old_completer) + readline.set_completer_delims(old_delims) + diff --git a/gomill/gomill/gtp_games.py b/gomill/gomill/gtp_games.py new file mode 100644 index 0000000..e1b0a2b --- /dev/null +++ b/gomill/gomill/gtp_games.py @@ -0,0 +1,816 @@ +"""Run a game between two GTP engines.""" + +from gomill import __version__ +from gomill.utils import * +from gomill.common import * +from gomill import gtp_controller +from gomill import handicap_layout +from gomill import boards +from gomill import sgf +from gomill.gtp_controller import BadGtpResponse, GtpChannelError + +class Game_result(object): + """Description of a game result. + + Public attributes: + players -- map colour -> player code + player_b -- player code + player_w -- player code + winning_player -- player code or None + losing_player -- player code or None + winning_colour -- 'b', 'w', or None + losing_colour -- 'b', 'w', or None + is_jigo -- bool + is_forfeit -- bool + sgf_result -- string describing the game's result (for sgf RE) + detail -- additional information (string or None) + game_id -- string or None + cpu_times -- map player code -> float or None or '?'. + + Winning/losing colour and player are None for a jigo, unknown result, or + void game. + + cpu_times are user time + system time. '?' means that gomill-cpu_time gave + an error. + + Game_results are suitable for pickling. + + """ + def __init__(self, players, winning_colour): + self.players = players.copy() + self.player_b = players['b'] + self.player_w = players['w'] + self.winning_colour = winning_colour + self.winning_player = players.get(winning_colour) + self.is_jigo = False + self.is_forfeit = False + self.game_id = None + if winning_colour is None: + self.sgf_result = "?" + else: + self.sgf_result = "%s+" % winning_colour.upper() + self.detail = None + self.cpu_times = {self.player_b : None, self.player_w : None} + + def __getstate__(self): + return ( + self.player_b, + self.player_w, + self.winning_colour, + self.sgf_result, + self.detail, + self.is_forfeit, + self.game_id, + self.cpu_times, + ) + + def __setstate__(self, state): + (self.player_b, + self.player_w, + self.winning_colour, + self.sgf_result, + self.detail, + self.is_forfeit, + self.game_id, + self.cpu_times, + ) = state + self.players = {'b' : self.player_b, 'w' : self.player_w} + self.winning_player = self.players.get(self.winning_colour) + self.is_jigo = (self.sgf_result == "0") + + def set_jigo(self): + self.sgf_result = "0" + self.is_jigo = True + + @property + def losing_colour(self): + if self.winning_colour is None: + return None + return opponent_of(self.winning_colour) + + @property + def losing_player(self): + if self.winning_colour is None: + return None + return self.players.get(opponent_of(self.winning_colour)) + + def describe(self): + """Return a short human-readable description of the result.""" + if self.winning_colour is not None: + s = "%s beat %s " % (self.winning_player, self.losing_player) + else: + s = "%s vs %s " % (self.players['b'], self.players['w']) + if self.is_jigo: + s += "jigo" + else: + s += self.sgf_result + if self.detail is not None: + s += " (%s)" % self.detail + return s + + def __repr__(self): + return "" % self.describe() + + +class Game(object): + """A single game between two GTP engines. + + Instantiate with: + board_size -- int + komi -- float (default 0.0) + move_limit -- int (default 1000) + + The 'commands' values are lists of strings, as for subprocess.Popen. + + Normal use: + + game = Game(...) + game.set_player_code('b', ...) + game.set_player_code('w', ...) + game.use_internal_scorer() or game.allow_scorer(...) [optional] + game.set_move_callback...() [optional] + game.set_player_subprocess('b', ...) or set_player_controller('b', ...) + game.set_player_subprocess('w', ...) or set_player_controller('w', ...) + game.request_engine_descriptions() [optional] + game.ready() + game.set_handicap(...) [optional] + game.run() + game.close_players() + game.make_sgf() or game.write_sgf(...) [optional] + + then retrieve the Game_result and moves. + + If neither use_internal_scorer() nor allow_scorer() is called, the game + won't be scored. + + Public attributes for reading: + players -- map colour -> player code + game_id -- string or None + result -- Game_result (None before the game is complete) + moves -- list of tuples (colour, move, comment) + move is a pair (row, col), or None for a pass + player_scores -- map player code -> string or None + engine_names -- map player code -> string + engine_descriptions -- map player code -> string + + player_scores values are the response to the final_score GTP command (if the + player was asked). + + Methods which communicate with engines may raise BadGtpResponse if the + engine returns a failure response. + + Methods which communicate with engines will normally raise GtpChannelError + if there is trouble communicating with the engine. But after the game result + has been decided, they will set these errors aside; retrieve them with + describe_late_errors(). + + This enforces a simple ko rule, but no superko rule. It accepts self-capture + moves. + + """ + + def __init__(self, board_size, komi=0.0, move_limit=1000): + self.players = {'b' : 'b', 'w' : 'w'} + self.game_id = None + self.controllers = {} + self.claim_allowed = {'b' : False, 'w' : False} + self.after_move_callback = None + self.board_size = board_size + self.komi = komi + self.move_limit = move_limit + self.allowed_scorers = [] + self.internal_scorer = False + self.handicap_compensation = "no" + self.handicap = 0 + self.first_player = "b" + self.engine_names = {} + self.engine_descriptions = {} + self.moves = [] + self.player_scores = {'b' : None, 'w' : None} + self.additional_sgf_props = [] + self.late_errors = [] + self.handicap_stones = None + self.result = None + self.board = boards.Board(board_size) + self.simple_ko_point = None + + + ## Configuration methods (callable before set_player_...) + + def set_player_code(self, colour, player_code): + """Specify a player code. + + player_code -- short ascii string + + The player codes are used to identify the players in game results, sgf + files, and the error messages. + + Setting these is optional but strongly encouraged. If not explicitly + set, they will just be 'b' and 'w'. + + Raises ValueError if both players are given the same code. + + """ + s = str(player_code) + if self.players[opponent_of(colour)] == s: + raise ValueError("player codes must be distinct") + self.players[colour] = s + + def set_game_id(self, game_id): + """Specify a game id. + + game_id -- string + + The game id is reported in the game result, and used as a default game + name in the SGF file. + + If you don't set it, it will have value None. + + """ + self.game_id = str(game_id) + + def use_internal_scorer(self, handicap_compensation='no'): + """Set the scoring method to internal. + + The internal scorer uses area score, assuming all stones alive. + + handicap_compensation -- 'no' (default), 'short', or 'full'. + + If handicap_compensation is 'full', one point is deducted from Black's + score for each handicap stone; if handicap_compensation is 'short', one + point is deducted from Black's score for each handicap stone except the + first. (The number of handicap stones is taken from the parameter to + set_handicap().) + + """ + self.internal_scorer = True + if handicap_compensation not in ('no', 'short', 'full'): + raise ValueError("bad handicap_compensation value: %s" % + handicap_compensation) + self.handicap_compensation = handicap_compensation + + def allow_scorer(self, colour): + """Allow the specified player to score the game. + + If this is called for both colours, both are asked to score. + + """ + self.allowed_scorers.append(colour) + + def set_claim_allowed(self, colour, b=True): + """Allow the specified player to claim a win. + + This will have no effect if the engine doesn't implement + gomill-genmove_ex. + + """ + self.claim_allowed[colour] = bool(b) + + def set_move_callback(self, fn): + """Specify a callback function to be called after every move. + + This function is called after each move is played, including passes but + not resignations, and not moves which triggered a forfeit. + + It is passed three parameters: colour, move, board + move is a pair (row, col), or None for a pass + + Treat the board parameter as read-only. + + Exceptions raised from the callback will be propagated unchanged out of + run(). + + """ + self.after_move_callback = fn + + + ## Channel methods + + def set_player_controller(self, colour, controller, + check_protocol_version=True): + """Specify a player using a Gtp_controller. + + controller -- Gtp_controller + check_protocol_version -- bool (default True) + + By convention, the channel name should be 'player '. + + If check_protocol_version is true, rejects an engine that declares a + GTP protocol version <> 2. + + Propagates GtpChannelError if there's an error checking the protocol + version. + + """ + self.controllers[colour] = controller + if check_protocol_version: + controller.check_protocol_version() + + def set_player_subprocess(self, colour, command, + check_protocol_version=True, **kwargs): + """Specify the a player as a subprocess. + + command -- list of strings (as for subprocess.Popen) + check_protocol_version -- bool (default True) + + Additional keyword arguments are passed to the Subprocess_gtp_channel + constructor. + + If check_protocol_version is true, rejects an engine that declares a + GTP protocol version <> 2. + + Propagates GtpChannelError if there's an error creating the + subprocess or checking the protocol version. + + """ + try: + channel = gtp_controller.Subprocess_gtp_channel(command, **kwargs) + except GtpChannelError, e: + raise GtpChannelError( + "error starting subprocess for player %s:\n%s" % + (self.players[colour], e)) + controller = gtp_controller.Gtp_controller( + channel, "player %s" % self.players[colour]) + self.set_player_controller(colour, controller, check_protocol_version) + + def get_controller(self, colour): + """Return the underlying Gtp_controller for the specified engine.""" + return self.controllers[colour] + + def send_command(self, colour, command, *arguments): + """Send the specified GTP command to one of the players. + + colour -- player to talk to ('b' or 'w') + command -- gtp command name (string) + arguments -- gtp arguments (strings) + + Returns the response as a string. + + Raises BadGtpResponse if the engine returns a failure response. + + You can use this at any time between set_player_...() and + close_players(). + + """ + return self.controllers[colour].do_command(command, *arguments) + + def maybe_send_command(self, colour, command, *arguments): + """Send the specified GTP command, if supported. + + Variant of send_command(): if the command isn't supported by the + engine, or gives a failure response, returns None. + + """ + controller = self.controllers[colour] + if controller.known_command(command): + try: + result = controller.do_command(command, *arguments) + except BadGtpResponse: + result = None + else: + result = None + return result + + def known_command(self, colour, command): + """Check whether the specified GTP command is supported.""" + return self.controllers[colour].known_command(command) + + def close_players(self): + """Close both controllers (if they're open). + + Retrieves the late errors for describe_late_errors(). + + If cpu times are not already set in the game result, sets them from the + CPU usage of the engine subprocesses. + + """ + for colour in ("b", "w"): + controller = self.controllers.get(colour) + if controller is None: + continue + controller.safe_close() + self.late_errors += controller.retrieve_error_messages() + self.update_cpu_times_from_channels() + + + ## High-level methods + + def request_engine_descriptions(self): + """Obtain the engines' name, version, and description by GTP. + + After you have called this, you can retrieve the results from the + engine_names and engine_descriptions attributes. + + If this has been called, other methods will use the engine name and/or + description when appropriate (ie, call this if you want proper engine + names to appear in the SGF file). + + """ + for colour in "b", "w": + controller = self.controllers[colour] + player = self.players[colour] + short_s, long_s = gtp_controller.describe_engine(controller, player) + self.engine_names[player] = short_s + self.engine_descriptions[player] = long_s + + def ready(self): + """Reset the engines' GTP game state (board size, contents, komi).""" + for colour in "b", "w": + controller = self.controllers[colour] + controller.do_command("boardsize", str(self.board_size)) + controller.do_command("clear_board") + controller.do_command("komi", str(self.komi)) + + def set_handicap(self, handicap, is_free): + """Initialise the board position for a handicap. + + Raises ValueError if the number of stones isn't valid (see GTP spec). + + Raises BadGtpResponse if there's an invalid respone to + place_free_handicap or fixed_handicap. + + """ + if is_free: + max_points = handicap_layout.max_free_handicap_for_board_size( + self.board_size) + if not 2 <= handicap < max_points: + raise ValueError + vertices = self.send_command( + "b", "place_free_handicap", str(handicap)) + try: + points = [move_from_vertex(vt, self.board_size) + for vt in vertices.split(" ")] + if None in points: + raise ValueError("response included 'pass'") + if len(set(points)) < len(points): + raise ValueError("duplicate point") + except ValueError, e: + raise BadGtpResponse( + "invalid response from place_free_handicap command " + "to %s: %s" % (self.players["b"], e)) + vertices = [format_vertex(point) for point in points] + self.send_command("w", "set_free_handicap", *vertices) + else: + # May propagate ValueError + points = handicap_layout.handicap_points(handicap, self.board_size) + for colour in "b", "w": + vertices = self.send_command( + colour, "fixed_handicap", str(handicap)) + try: + seen_points = [move_from_vertex(vt, self.board_size) + for vt in vertices.split(" ")] + if set(seen_points) != set(points): + raise ValueError + except ValueError: + raise BadGtpResponse( + "bad response from fixed_handicap command " + "to %s: %s" % (self.players[colour], vertices)) + self.board.apply_setup(points, [], []) + self.handicap = handicap + self.additional_sgf_props.append(('HA', handicap)) + self.handicap_stones = points + self.first_player = "w" + + def _forfeit_to(self, winner, msg): + self.winner = winner + self.forfeited = True + self.forfeit_reason = msg + + def _play_move(self, colour): + opponent = opponent_of(colour) + if (self.claim_allowed[colour] and + self.known_command(colour, "gomill-genmove_ex")): + genmove_command = ["gomill-genmove_ex", colour, "claim"] + may_claim = True + else: + genmove_command = ["genmove", colour] + may_claim = False + try: + move_s = self.send_command(colour, *genmove_command).lower() + except BadGtpResponse, e: + self._forfeit_to(opponent, str(e)) + return + if move_s == "resign": + self.winner = opponent + self.seen_resignation = True + return + if may_claim and move_s == "claim": + self.winner = colour + self.seen_claim = True + return + try: + move = move_from_vertex(move_s, self.board_size) + except ValueError: + self._forfeit_to(opponent, "%s attempted ill-formed move %s" % ( + self.players[colour], move_s)) + return + comment = self.maybe_send_command(colour, "gomill-explain_last_move") + comment = sanitise_utf8(comment) + if comment == "": + comment = None + if move is not None: + self.pass_count = 0 + if move == self.simple_ko_point: + self._forfeit_to( + opponent, "%s attempted move to ko-forbidden point %s" % ( + self.players[colour], move_s)) + return + row, col = move + try: + self.simple_ko_point = self.board.play(row, col, colour) + except ValueError: + self._forfeit_to( + opponent, "%s attempted move to occupied point %s" % ( + self.players[colour], move_s)) + return + else: + self.pass_count += 1 + self.simple_ko_point = None + try: + self.send_command(opponent, "play", colour, move_s) + except BadGtpResponse, e: + if e.gtp_error_message == "illegal move": + # we assume the move really was illegal, so 'colour' should lose + self._forfeit_to(opponent, "%s claims move %s is illegal" % ( + self.players[opponent], move_s)) + else: + self._forfeit_to(colour, str(e)) + return + self.moves.append((colour, move, comment)) + if self.after_move_callback: + self.after_move_callback(colour, move, self.board) + + def run(self): + """Run a complete game between the two players. + + Sets self.moves and self.result. + + Sets CPU times in the game result if available via GTP. + + """ + self.pass_count = 0 + self.winner = None + self.margin = None + self.scorers_disagreed = False + self.seen_resignation = False + self.seen_claim = False + self.forfeited = False + self.hit_move_limit = False + self.forfeit_reason = None + self.passed_out = False + player = self.first_player + move_count = 0 + while move_count < self.move_limit: + self._play_move(player) + if self.pass_count == 2: + self.passed_out = True + self.winner, self.margin, self.scorers_disagreed = \ + self._score_game() + break + if self.winner is not None: + break + player = opponent_of(player) + move_count += 1 + else: + self.hit_move_limit = True + self.calculate_result() + self.calculate_cpu_times() + + def fake_run(self, winner): + """Set state variables as if the game had been run (for testing). + + You don't need to use set_player_{subprocess,controller} to call this. + + winner -- 'b' or 'w' + + """ + self.winner = winner + self.seen_resignation = False + self.seen_claim = False + self.forfeited = False + self.hit_move_limit = False + self.forfeit_reason = None + self.passed_out = True + self.margin = True + self.scorers_disagreed = False + self.calculate_result() + + def _score_game(self): + is_disagreement = False + if self.internal_scorer: + score = self.board.area_score() - self.komi + if self.handicap_compensation == "full": + score -= self.handicap + elif self.handicap_compensation == "short": + score -= (self.handicap - 1) + if score > 0: + winner = "b" + margin = score + elif score < 0: + winner = "w" + margin = -score + else: + winner = None + margin = 0 + else: + winners = [] + margins = [] + for colour in self.allowed_scorers: + final_score = self.maybe_send_command(colour, "final_score") + if final_score is None: + continue + self.player_scores[colour] = final_score + final_score = final_score.upper() + if final_score == "0": + winners.append(None) + margins.append(0) + continue + if final_score.startswith("B+"): + winners.append("b") + elif final_score.startswith("W+"): + winners.append("w") + else: + continue + try: + margin = float(final_score[2:]) + if margin <= 0: + margin = None + except ValueError: + margin = None + margins.append(margin) + if len(set(winners)) == 1: + winner = winners[0] + if len(set(margins)) == 1: + margin = margins[0] + else: + margin = None + else: + if len(set(winners)) > 1: + is_disagreement = True + winner = None + margin = None + return winner, margin, is_disagreement + + def calculate_result(self): + """Set self.result. + + You shouldn't normally call this directly. + + """ + result = Game_result(self.players, self.winner) + result.game_id = self.game_id + if self.hit_move_limit: + result.sgf_result = "Void" + result.detail = "hit move limit" + elif self.seen_resignation: + result.sgf_result += "R" + elif self.seen_claim: + # Leave SGF result in form 'B+' + result.detail = "claim" + elif self.forfeited: + result.sgf_result += "F" + result.is_forfeit = True + result.detail = "forfeit: %s" % self.forfeit_reason + else: + assert self.passed_out + if self.winner is None: + if self.margin == 0: + result.set_jigo() + elif self.scorers_disagreed: + result.detail = "players disagreed" + else: + result.detail = "no score reported" + elif self.margin is not None: + result.sgf_result += format_float(self.margin) + else: + # Players returned something like 'B+?', + # or disagreed about the margin + # Leave SGF result in form 'B+' + result.detail = "unknown margin" + self.result = result + + def calculate_cpu_times(self): + """Set CPU times in self.result. + + You shouldn't normally call this directly. + + """ + # The ugliness with cpu_time '?' is to avoid using the cpu time reported + # by channel close() for engines which claim to support gomill-cpu_time + # but give an error. + for colour in ('b', 'w'): + cpu_time = None + controller = self.controllers[colour] + if controller.safe_known_command('gomill-cpu_time'): + try: + s = controller.safe_do_command('gomill-cpu_time') + cpu_time = float(s) + except (BadGtpResponse, ValueError, TypeError): + cpu_time = "?" + self.result.cpu_times[self.players[colour]] = cpu_time + + def update_cpu_times_from_channels(self): + """Set CPU times in self.result from the channel resource usage. + + There's normally no need to call this directly: close_players() will do + it. + + Has no effect if CPU times have already been set. + + """ + for colour in ('b', 'w'): + controller = self.controllers.get(colour) + if controller is None: + continue + ru = controller.channel.resource_usage + if (ru is not None and self.result is not None and + self.result.cpu_times[self.players[colour]] is None): + self.result.cpu_times[self.players[colour]] = \ + ru.ru_utime + ru.ru_stime + + def describe_late_errors(self): + """Retrieve the late error messages. + + Returns a string, or None if there were no late errors. + + This is only available after close_players() has been called. + + The late errors are low-level errors which occurred after the game + result was decided and so were set asied. In particular, they include + any errors from closing (including failure responses from the final + 'quit' command) + + """ + if not self.late_errors: + return None + return "\n".join(self.late_errors) + + def describe_scoring(self): + """Return a multiline string describing the game's scoring.""" + if self.result is None: + return "" + def normalise_score(s): + s = s.upper() + if s.endswith(".0"): + s = s[:-2] + return s + l = [self.result.describe()] + sgf_result = self.result.sgf_result + score_b = self.player_scores['b'] + score_w = self.player_scores['w'] + if ((score_b is not None and normalise_score(score_b) != sgf_result) or + (score_w is not None and normalise_score(score_w) != sgf_result)): + for colour, score in (('b', score_b), ('w', score_w)): + if score is not None: + l.append("%s final_score: %s" % + (self.players[colour], score)) + return "\n".join(l) + + def make_sgf(self, game_end_message=None): + """Return an SGF description of the game. + + Returns an Sgf_game object. + + game_end_message -- optional string to put in the final comment. + + If game_end_message is specified, it appears before the text describing + 'late errors'. + + """ + sgf_game = sgf.Sgf_game(self.board_size) + root = sgf_game.get_root() + root.set('KM', self.komi) + root.set('AP', ("gomill", __version__)) + for prop, value in self.additional_sgf_props: + root.set(prop, value) + sgf_game.set_date() + if self.engine_names: + root.set('PB', self.engine_names[self.players['b']]) + root.set('PW', self.engine_names[self.players['w']]) + if self.game_id: + root.set('GN', self.game_id) + if self.handicap_stones: + root.set_setup_stones(black=self.handicap_stones, white=[]) + for colour, move, comment in self.moves: + node = sgf_game.extend_main_sequence() + node.set_move(colour, move) + if comment is not None: + node.set("C", comment) + last_node = sgf_game.get_last_node() + if self.result is not None: + root.set('RE', self.result.sgf_result) + last_node.add_comment_text(self.describe_scoring()) + if game_end_message is not None: + last_node.add_comment_text(game_end_message) + late_error_messages = self.describe_late_errors() + if late_error_messages is not None: + last_node.add_comment_text(late_error_messages) + return sgf_game + + def write_sgf(self, pathname, game_end_message=None): + """Write an SGF description of the game to the specified pathname.""" + sgf_game = self.make_sgf(game_end_message) + f = open(pathname, "w") + f.write(sgf_game.serialise()) + f.close() + diff --git a/gomill/gomill/gtp_proxy.py b/gomill/gomill/gtp_proxy.py new file mode 100644 index 0000000..002fae8 --- /dev/null +++ b/gomill/gomill/gtp_proxy.py @@ -0,0 +1,262 @@ +"""Support for implementing proxy GTP engines. + +That is, engines which implement some or all of their commands by sending them +on to another engine (the _back end_). + +""" + +from gomill import gtp_controller +from gomill import gtp_engine +from gomill.gtp_controller import ( + BadGtpResponse, GtpChannelError, GtpChannelClosed) +from gomill.gtp_engine import GtpError, GtpQuit, GtpFatalError + + +class BackEndError(StandardError): + """Difficulty communicating with the back end. + + Public attributes: + cause -- Exception instance of an underlying exception (or None) + + """ + def __init__(self, args, cause=None): + StandardError.__init__(self, args) + self.cause = cause + +class Gtp_proxy(object): + """Manager for a GTP proxy engine. + + Public attributes: + engine -- Gtp_engine_protocol + controller -- Gtp_controller + + The 'engine' attribute is the proxy engine. Initially it supports all the + commands reported by the back end's 'list_commands'. You can add commands to + it in the usual way; new commands will override any commands with the same + names in the back end. + + The proxy engine also supports the following commands: + gomill-passthrough [args] ... + Run a command on the back end (use this to get at overridden commands, + or commands which don't appear in list_commands) + + If the proxy subprocess exits, this will be reported (as a transport error) + when the next command is sent. If you're using handle_command, it will + apropriately turn this into a fatal error. + + Sample use: + proxy = gtp_proxy.Gtp_proxy() + proxy.set_back_end_subprocess([, , ...]) + proxy.engine.add_command(...) + try: + proxy.run() + except KeyboardInterrupt: + sys.exit(1) + + The default 'quit' handler passes 'quit' on the back end and raises + GtpQuit. + + If you add a handler which you expect to cause the back end to exit (eg, by + sending it 'quit'), you should have call expect_back_end_exit() (and usually + also raise GtpQuit). + + If you want to hide one of the underlying commands, or don't want one of the + additional commands, just use engine.remove_command(). + + """ + def __init__(self): + self.controller = None + self.engine = None + + def _back_end_is_set(self): + return self.controller is not None + + def _make_back_end_handlers(self): + result = {} + for command in self.back_end_commands: + def handler(args, _command=command): + return self.handle_command(_command, args) + result[command] = handler + return result + + def _make_engine(self): + self.engine = gtp_engine.Gtp_engine_protocol() + self.engine.add_commands(self._make_back_end_handlers()) + self.engine.add_protocol_commands() + self.engine.add_commands({ + 'quit' : self.handle_quit, + 'gomill-passthrough' : self.handle_passthrough, + }) + + def set_back_end_controller(self, controller): + """Specify the back end using a Gtp_controller. + + controller -- Gtp_controller + + Raises BackEndError if it can't communicate with the back end. + + By convention, the controller's channel name should be "back end". + + """ + if self._back_end_is_set(): + raise StandardError("back end already set") + self.controller = controller + try: + self.back_end_commands = controller.list_commands() + except (GtpChannelError, BadGtpResponse), e: + raise BackEndError(str(e), cause=e) + self._make_engine() + + def set_back_end_subprocess(self, command, **kwargs): + """Specify the back end as a subprocess. + + command -- list of strings (as for subprocess.Popen) + + Additional keyword arguments are passed to the Subprocess_gtp_channel + constructor. + + Raises BackEndError if it can't communicate with the back end. + + """ + try: + channel = gtp_controller.Subprocess_gtp_channel(command, **kwargs) + except GtpChannelError, e: + # Probably means exec failure + raise BackEndError("can't launch back end command\n%s" % e, cause=e) + controller = gtp_controller.Gtp_controller(channel, "back end") + self.set_back_end_controller(controller) + + def close(self): + """Close the channel to the back end. + + It's safe to call this at any time after set_back_end_... (including + after receiving a BackEndError). + + It's not strictly necessary to call this if you're going to exit from + the parent process anyway, as that will naturally close the command + channel. But some engines don't behave well if you don't send 'quit', + so it's safest to close the proxy explicitly. + + This will send 'quit' if low-level errors have not previously been seen + on the channel, unless expect_back_end_exit() has been called. + + Errors (including failure responses to 'quit') are reported by raising + BackEndError. + + """ + if self.controller is None: + return + self.controller.safe_close() + late_errors = self.controller.retrieve_error_messages() + if late_errors: + raise BackEndError("\n".join(late_errors)) + + def run(self): + """Run a GTP session on stdin and stdout, using the proxy engine. + + This is provided for convenience; it's also ok to use the proxy engine + directly. + + Returns either when EOF is seen on stdin, or when a handler (such as the + default 'quit' handler) raises GtpQuit. + + Closes the channel to the back end before it returns. When it is + meaningful (eg, for subprocess channels) this waits for the back end to + exit. + + Propagates ControllerDisconnected if a pipe connected to stdout goes + away. + + """ + gtp_engine.run_interactive_gtp_session(self.engine) + self.close() + + def pass_command(self, command, args): + """Pass a command to the back end, and return its response. + + The response (or failure response) is unchanged, except for whitespace + normalisation. + + This passes the command to the back end even if it isn't included in the + back end's list_commands output; the back end will presumably return an + 'unknown command' error. + + Failure responses from the back end are reported by raising + BadGtpResponse. + + Low-level (ie, transport or protocol) errors are reported by raising + BackEndError. + + """ + if not self._back_end_is_set(): + raise StandardError("back end isn't set") + try: + return self.controller.do_command(command, *args) + except GtpChannelError, e: + raise BackEndError(str(e), cause=e) + + def handle_command(self, command, args): + """Run a command on the back end, from inside a GTP handler. + + This is a variant of pass_command, intended to be used directly in a + command handler. + + Failure responses from the back end are reported by raising GtpError. + + Low-level (ie, transport or protocol) errors are reported by raising + GtpFatalError. + + """ + try: + return self.pass_command(command, args) + except BadGtpResponse, e: + raise GtpError(e.gtp_error_message) + except BackEndError, e: + raise GtpFatalError(str(e)) + + def back_end_has_command(self, command): + """Say whether the back end supports the specified command. + + This uses known_command, not list_commands. It caches the results. + + Low-level (ie, transport or protocol) errors are reported by raising + BackEndError. + + """ + if not self._back_end_is_set(): + raise StandardError("back end isn't set") + try: + return self.controller.known_command(command) + except GtpChannelError, e: + raise BackEndError(str(e), cause=e) + + def expect_back_end_exit(self): + """Mark that the back end is expected to have exited. + + Call this from any handler which you expect to cause the back end to + exit (eg, by sending it 'quit'). + + """ + self.controller.channel_is_bad = True + + def handle_quit(self, args): + # Ignores GtpChannelClosed + try: + result = self.pass_command("quit", []) + except BackEndError, e: + if isinstance(e.cause, GtpChannelClosed): + result = "" + else: + raise GtpFatalError(str(e)) + except BadGtpResponse, e: + self.expect_back_end_exit() + raise GtpFatalError(e.gtp_error_message) + self.expect_back_end_exit() + raise GtpQuit(result) + + def handle_passthrough(self, args): + try: + command = args[0] + except IndexError: + gtp_engine.report_bad_arguments() + return self.handle_command(command, args[1:]) diff --git a/gomill/gomill/gtp_states.py b/gomill/gomill/gtp_states.py new file mode 100644 index 0000000..1fc92d7 --- /dev/null +++ b/gomill/gomill/gtp_states.py @@ -0,0 +1,651 @@ +"""Stateful GTP engine.""" + +from __future__ import with_statement + +from gomill import __version__ +from gomill.common import * +from gomill import ascii_boards +from gomill import boards +from gomill import gtp_engine +from gomill import handicap_layout +from gomill import sgf +from gomill import sgf_grammar +from gomill import sgf_moves +from gomill.gtp_engine import GtpError + + +class History_move(object): + """Information about a move (for move_history). + + Public attributes: + colour + move -- (row, col), or None for a pass + comments -- multiline string, or None + cookie + + comments are used by gomill-savesgf. + + The cookie attribute stores an arbitrary value which was provided by the + move generator when the move was played. The cookie attribute of a move + which did not come from the move generator is None. + + This is a way for a move generator to maintain state across moves, without + becoming confused by 'undo' &c. It's not intended for storing large amounts + of data. + + """ + def __init__(self, colour, move, comments=None, cookie=None): + self.colour = colour + self.move = move + self.comments = comments + self.cookie = cookie + + def is_pass(self): + return (self.move is None) + + +class Game_state(object): + """Data passed to a move generator. + + Public attributes: + size -- int + board -- boards.Board + komi -- float + history_base -- boards.Board + move_history -- list of History_move objects + ko_point -- (row, col) or None + handicap -- int >= 2 or None + for_regression -- bool + time_settings -- tuple (m, b, s), or None + time_remaining -- int (seconds), or None + canadian_stones_remaining -- int or None + + 'board' represents the current board position. + + history_base represents a (possibly) earlier board position; move_history + lists the moves leading to 'board' from that position. + + Normally, history_base will be an empty board, or else be the position after + the placement of handicap stones; but if the loadsgf command has been used + it may be the position given by setup stones in the SGF file. + + The get_last_move() and get_last_move_and_cookie() functions below are + provided to help interpret move history. + + + ko_point is the point forbidden by the simple ko rule. This is provided for + convenience for engines which don't want to deduce it from the move history. + To handle superko properly, engines will have to use the move history. + + + 'handicap' is provided in case the engine wants to modify its behaviour in + handicap games; it can safely be ignored. Any handicap stones will be + present in history_base. + + + for_regression is true if the command was 'reg_genmove'; engines which care + should use a fixed seed in this case. + + + time_settings describes the time limits for the game; time_remaining + describes the current situation. + + time_settings values m, b, s are main time (in seconds), 'Canadian byo-yomi' + time (in seconds), and 'Canadian byo-yomi' stones; see GTP spec 4.2 (which + describes what 0 values mean). time_settings None means the information + isn't available. + + time_remaining None means the game isn't timed. canadian_stones_remaining + None means we're in main time. + + The most important information for the move generator is in time_remaining; + time_settings lets it know whether it's going to get overtime as well. It's + possible for time_remaining to be available but not time_settings (if the + controller doesn't send time_settings). + + """ + +class Move_generator_result(object): + """Return value from a move generator. + + Public attributes: + resign -- bool + pass_move -- bool + move -- (row, col), or None + claim -- bool (for gomill-genmove_ex claim) + comments -- multiline string, or None + cookie -- arbitrary value + + Exactly one of the first three attributes should be set to a nondefault + value. The other attributes can be safely left at their defaults. + + If claim is true, either 'move' or 'pass_move' must still be set. + + comments are used by gomill-explain_last_move and gomill-savesgf. + + See History_move for an explanation of the cookie attribute. It has the + value None if not explicitly set. + + """ + def __init__(self): + self.resign = False + self.pass_move = False + self.move = None + self.claim = False + self.comments = None + self.cookie = None + + +class Gtp_state(object): + """Manage the stateful part of the GTP engine protocol. + + This supports implementing a GTP engine using a stateless move generator. + + Sample use: + gtp_state = Gtp_state(...) + engine = Gtp_engine_protocol() + engine.add_commands(gtp_state.get_handlers()) + + A Gtp_state maintains the following state: + board configuration + move history + komi + simple ko ban + + + Instantiate with a _move generator function_ and a list of acceptable board + sizes (default 19 only). + + The move generator function is called to handle genmove. It is passed + arguments (game_state, colour to play). It should return a + Move_generator_result. It must not modify data passed in the game_state. + + If the move generator returns an occupied point, Gtp_state will report a GTP + error. Gtp_state does not enforce any ko rule. It permits self-captures. + + """ + + def __init__(self, move_generator, acceptable_sizes=None): + self.komi = 0.0 + self.time_settings = None + self.time_status = { + 'b' : (None, None), + 'w' : (None, None), + } + self.move_generator = move_generator + if acceptable_sizes is None: + self.acceptable_sizes = set((19,)) + self.board_size = 19 + else: + self.acceptable_sizes = set(acceptable_sizes) + self.board_size = min(self.acceptable_sizes) + self.reset() + + def reset(self): + self.board = boards.Board(self.board_size) + # None, or a small integer + self.handicap = None + self.simple_ko_point = None + # Player that any simple_ko_point is banned for + self.simple_ko_player = None + self.history_base = boards.Board(self.board_size) + # list of History_move objects + self.move_history = [] + + def set_history_base(self, board): + """Change the history base to a new position. + + Takes ownership of 'board'. + + Clears the move history. + + """ + self.history_base = board + self.move_history = [] + + def reset_to_moves(self, history_moves): + """Reset to history base and play the specified moves. + + history_moves -- list of History_move objects. + + 'history_moves' becomes the new move history. Takes ownership of + 'history_moves'. + + Raises ValueError if there is an invalid move in the list. + + """ + self.board = self.history_base.copy() + simple_ko_point = None + simple_ko_player = None + for history_move in history_moves: + if history_move.is_pass(): + self.simple_ko_point = None + continue + row, col = history_move.move + # Propagates ValueError if the move is bad + simple_ko_point = self.board.play(row, col, history_move.colour) + simple_ko_player = opponent_of(history_move.colour) + self.simple_ko_point = simple_ko_point + self.simple_ko_player = simple_ko_player + self.move_history = history_moves + + def set_komi(self, f): + max_komi = 625.0 + if f < -max_komi: + f = -max_komi + elif f > max_komi: + f = max_komi + self.komi = f + + def handle_boardsize(self, args): + try: + size = gtp_engine.interpret_int(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + if size not in self.acceptable_sizes: + raise GtpError("unacceptable size") + self.board_size = size + self.reset() + + def handle_clear_board(self, args): + self.reset() + + def handle_komi(self, args): + try: + f = gtp_engine.interpret_float(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + self.set_komi(f) + + def handle_fixed_handicap(self, args): + try: + number_of_stones = gtp_engine.interpret_int(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + if not self.board.is_empty(): + raise GtpError("board not empty") + try: + points = handicap_layout.handicap_points( + number_of_stones, self.board_size) + except ValueError: + raise GtpError("invalid number of stones") + for row, col in points: + self.board.play(row, col, 'b') + self.simple_ko_point = None + self.handicap = number_of_stones + self.set_history_base(self.board.copy()) + return " ".join(format_vertex((row, col)) + for (row, col) in points) + + def handle_set_free_handicap(self, args): + max_points = handicap_layout.max_free_handicap_for_board_size( + self.board_size) + if not 2 <= len(args) <= max_points: + raise GtpError("invalid number of stones") + if not self.board.is_empty(): + raise GtpError("board not empty") + try: + for vertex_s in args: + move = gtp_engine.interpret_vertex(vertex_s, self.board_size) + if move is None: + raise GtpError("'pass' not permitted") + row, col = move + try: + self.board.play(row, col, 'b') + except ValueError: + raise GtpError("engine error: %s is occupied" % vertex_s) + except Exception: + self.reset() + raise + self.set_history_base(self.board.copy()) + self.handicap = len(args) + self.simple_ko_point = None + + def _choose_free_handicap_moves(self, number_of_stones): + i = min(number_of_stones, + handicap_layout.max_fixed_handicap_for_board_size( + self.board_size)) + return handicap_layout.handicap_points(i, self.board_size) + + def handle_place_free_handicap(self, args): + try: + number_of_stones = gtp_engine.interpret_int(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + max_points = handicap_layout.max_free_handicap_for_board_size( + self.board_size) + if not 2 <= number_of_stones <= max_points: + raise GtpError("invalid number of stones") + if not self.board.is_empty(): + raise GtpError("board not empty") + if number_of_stones == max_points: + number_of_stones = max_points - 1 + moves = self._choose_free_handicap_moves(number_of_stones) + try: + try: + if len(moves) > number_of_stones: + raise ValueError + for row, col in moves: + self.board.play(row, col, 'b') + except (ValueError, TypeError): + raise GtpError("invalid result from move generator: %s" + % format_vertex_list(moves)) + except Exception: + self.reset() + raise + self.simple_ko_point = None + self.handicap = number_of_stones + self.set_history_base(self.board.copy()) + return " ".join(format_vertex((row, col)) + for (row, col) in moves) + + def handle_play(self, args): + try: + colour_s, vertex_s = args[:2] + except ValueError: + gtp_engine.report_bad_arguments() + colour = gtp_engine.interpret_colour(colour_s) + move = gtp_engine.interpret_vertex(vertex_s, self.board_size) + if move is None: + self.simple_ko_point = None + self.move_history.append(History_move(colour, None)) + return + row, col = move + try: + self.simple_ko_point = self.board.play(row, col, colour) + self.simple_ko_player = opponent_of(colour) + except ValueError: + raise GtpError("illegal move") + self.move_history.append(History_move(colour, move)) + + def handle_showboard(self, args): + return "\n%s\n" % ascii_boards.render_board(self.board) + + def _handle_genmove(self, args, for_regression=False, allow_claim=False): + """Common implementation for genmove commands.""" + try: + colour = gtp_engine.interpret_colour(args[0]) + except IndexError: + gtp_engine.report_bad_arguments() + game_state = Game_state() + game_state.size = self.board_size + game_state.board = self.board + game_state.history_base = self.history_base + game_state.move_history = self.move_history + game_state.komi = self.komi + game_state.for_regression = for_regression + if self.simple_ko_point is not None and self.simple_ko_player == colour: + game_state.ko_point = self.simple_ko_point + else: + game_state.ko_point = None + game_state.handicap = self.handicap + game_state.time_settings = self.time_settings + game_state.time_remaining, game_state.canadian_stones_remaining = \ + self.time_status[colour] + generated = self.move_generator(game_state, colour) + if allow_claim and generated.claim: + return 'claim' + if generated.resign: + return 'resign' + if generated.pass_move: + if not for_regression: + self.move_history.append(History_move( + colour, None, generated.comments, generated.cookie)) + return 'pass' + row, col = generated.move + vertex = format_vertex((row, col)) + if not for_regression: + try: + self.simple_ko_point = self.board.play(row, col, colour) + self.simple_ko_player = opponent_of(colour) + except ValueError: + raise GtpError("engine error: tried to play %s" % vertex) + self.move_history.append( + History_move(colour, generated.move, + generated.comments, generated.cookie)) + return vertex + + def handle_genmove(self, args): + return self._handle_genmove(args) + + def handle_genmove_ex(self, args): + if not args: + return "claim" + allow_claim = False + for arg in args[1:]: + if arg == 'claim': + allow_claim = True + return self._handle_genmove(args[:1], allow_claim=allow_claim) + + def handle_reg_genmove(self, args): + return self._handle_genmove(args, for_regression=True) + + def handle_undo(self, args): + if not self.move_history: + raise GtpError("cannot undo") + try: + self.reset_to_moves(self.move_history[:-1]) + except ValueError: + raise GtpError("corrupt history") + + def _load_file(self, pathname): + """Read the specified file and return its contents as a string. + + Subclasses can override this to change how loadsgf interprets filenames. + + May raise EnvironmentError. + + """ + with open(pathname) as f: + return f.read() + + def handle_loadsgf(self, args): + try: + pathname = args[0] + except IndexError: + gtp_engine.report_bad_arguments() + if len(args) > 1: + move_number = gtp_engine.interpret_int(args[1]) + else: + move_number = None + # The GTP spec mandates the "cannot load file" error message, so we + # can't be more helpful. + try: + s = self._load_file(pathname) + except EnvironmentError: + raise GtpError("cannot load file") + try: + sgf_game = sgf.Sgf_game.from_string(s) + except ValueError: + raise GtpError("cannot load file") + new_size = sgf_game.get_size() + if new_size not in self.acceptable_sizes: + raise GtpError("unacceptable size") + self.board_size = new_size + try: + komi = sgf_game.get_komi() + except ValueError: + raise GtpError("bad komi") + try: + handicap = sgf_game.get_handicap() + except ValueError: + # Handicap isn't important, so soldier on + handicap = None + try: + sgf_board, plays = sgf_moves.get_setup_and_moves(sgf_game) + except ValueError, e: + raise GtpError(str(e)) + history_moves = [History_move(colour, move) + for (colour, move) in plays] + if move_number is None: + new_move_history = history_moves + else: + # gtp spec says we want the "position before move_number" + move_number = max(0, move_number-1) + new_move_history = history_moves[:move_number] + old_history_base = self.history_base + old_move_history = self.move_history + try: + self.set_history_base(sgf_board) + self.reset_to_moves(new_move_history) + except ValueError: + try: + self.set_history_base(old_history_base) + self.reset_to_moves(old_move_history) + except ValueError: + raise GtpError("bad move in file and corrupt history") + raise GtpError("bad move in file") + self.set_komi(komi) + self.handicap = handicap + + def handle_time_left(self, args): + # colour time stones + try: + colour = gtp_engine.interpret_colour(args[0]) + time_remaining = gtp_engine.interpret_int(args[1]) + stones_remaining = gtp_engine.interpret_int(args[2]) + except IndexError: + gtp_engine.report_bad_arguments() + if stones_remaining == 0: + stones_remaining = None + self.time_status[colour] = (time_remaining, stones_remaining) + + def handle_time_settings(self, args): + try: + main_time = gtp_engine.interpret_int(args[0]) + canadian_time = gtp_engine.interpret_int(args[1]) + canadian_stones = gtp_engine.interpret_int(args[2]) + except IndexError: + gtp_engine.report_bad_arguments() + self.time_settings = (main_time, canadian_time, canadian_stones) + + def handle_explain_last_move(self, args): + try: + return self.move_history[-1].comments + except IndexError: + return None + + def _save_file(self, pathname, contents): + """Write a string to the specified file. + + Subclasses can override this to change how gomill-savesgf interprets + filenames. + + May raise EnvironmentError. + + """ + with open(pathname, "w") as f: + f.write(contents) + + def handle_savesgf(self, args): + try: + pathname = args[0] + except IndexError: + gtp_engine.report_bad_arguments() + + sgf_game = sgf.Sgf_game(self.board_size) + root = sgf_game.get_root() + root.set('KM', self.komi) + root.set('AP', ("gomill", __version__)) + sgf_game.set_date() + if self.handicap is not None: + root.set('HA', self.handicap) + for arg in args[1:]: + try: + identifier, value = arg.split("=", 1) + if not identifier.isalpha(): + raise ValueError + identifier = identifier.upper() + value = value.replace("\\_", " ").replace("\\\\", "\\") + except Exception: + gtp_engine.report_bad_arguments() + root.set_raw(identifier, sgf_grammar.escape_text(value)) + sgf_moves.set_initial_position(sgf_game, self.history_base) + for history_move in self.move_history: + node = sgf_game.extend_main_sequence() + node.set_move(history_move.colour, history_move.move) + if history_move.comments is not None: + node.set("C", history_move.comments) + sgf_moves.indicate_first_player(sgf_game) + try: + self._save_file(pathname, sgf_game.serialise()) + except EnvironmentError, e: + raise GtpError("error writing file: %s" % e) + + + def get_handlers(self): + return {'boardsize' : self.handle_boardsize, + 'clear_board' : self.handle_clear_board, + 'komi' : self.handle_komi, + 'fixed_handicap' : self.handle_fixed_handicap, + 'set_free_handicap' : self.handle_set_free_handicap, + 'place_free_handicap' : self.handle_place_free_handicap, + 'play' : self.handle_play, + 'genmove' : self.handle_genmove, + 'gomill-genmove_ex' : self.handle_genmove_ex, + 'reg_genmove' : self.handle_reg_genmove, + 'undo' : self.handle_undo, + 'showboard' : self.handle_showboard, + 'loadsgf' : self.handle_loadsgf, + 'gomill-explain_last_move' : self.handle_explain_last_move, + 'gomill-savesgf' : self.handle_savesgf, + } + + def get_time_handlers(self): + """Return handlers for time-related commands. + + These are separated out so that engines which don't support time + handling can avoid advertising time support. + + """ + return {'time_left' : self.handle_time_left, + 'time_settings' : self.handle_time_settings, + } + + +def get_last_move(history_moves, player): + """Get the last move from the move history, checking it's by the opponent. + + This is a convenience function for use by move generators. + + history_moves -- list of History_move objects + player -- player to play current move ('b' or 'w') + + Returns a pair (move_is_available, move) + where move is (row, col), or None for a pass. + + If the last move is unknown, or it wasn't by the opponent, move_is_available + is False and move is None. + + """ + if not history_moves: + return False, None + if history_moves[-1].colour != opponent_of(player): + return False, None + return True, history_moves[-1].move + +def get_last_move_and_cookie(history_moves, player): + """Interpret recent move history. + + This is a convenience function for use by move generators. + + This is a variant of get_last_move, which also returns the last-but-one + move's cookie if available. + + Returns a tuple (move_is_available, opponent's move, cookie) + + move_is_available has the same meaning as for get_last_move(). + + If move_is_available is false, or if the next-to-last move is unavailable or + wasn't by the current player, cookie is None. + + """ + move_is_available, opponents_move = get_last_move(history_moves, player) + if (move_is_available and len(history_moves) > 1 and + history_moves[-2].colour == player): + cookie = history_moves[-2].cookie + else: + cookie = None + return move_is_available, opponents_move, cookie + + diff --git a/gomill/gomill/handicap_layout.py b/gomill/gomill/handicap_layout.py new file mode 100644 index 0000000..36910a4 --- /dev/null +++ b/gomill/gomill/handicap_layout.py @@ -0,0 +1,54 @@ +"""Standard layout of fixed handicap stones. + +This follows the rules from the GTP spec. + +""" + +def max_free_handicap_for_board_size(board_size): + """Return the maximum number of stones for place_free_handicap command.""" + return board_size * board_size - 1 + +def max_fixed_handicap_for_board_size(board_size): + """Return the maximum number of stones for fixed_handicap command.""" + if board_size <= 7: + return 0 + if board_size > 25: + raise ValueError + if board_size % 2 == 0 or board_size == 7: + return 4 + else: + return 9 + +handicap_pattern = [ + ['00', '22'], + ['00', '22', '20'], + ['00', '22', '20', '02'], + ['00', '22', '20', '02', '11'], + ['00', '22', '20', '02', '10', '12'], + ['00', '22', '20', '02', '10', '12', '11'], + ['00', '22', '20', '02', '10', '12', '01', '21'], + ['00', '22', '20', '02', '10', '12', '01', '21', '11'], +] + +def handicap_points(number_of_stones, board_size): + """Return the handicap points for a given number of stones and board size. + + Returns a list of pairs (row, col), length 'number_of_stones'. + + Raises ValueError if there isn't a placement pattern for the specified + number of handicap stones and board size. + + """ + if number_of_stones > max_fixed_handicap_for_board_size(board_size): + raise ValueError + if number_of_stones < 2: + raise ValueError + if board_size < 13: + altitude = 2 + else: + altitude = 3 + pos = {'0' : altitude, + '1' : (board_size - 1) / 2, + '2' : board_size - altitude - 1} + return [(pos[s[0]], pos[s[1]]) + for s in handicap_pattern[number_of_stones-2]] diff --git a/gomill/gomill/job_manager.py b/gomill/gomill/job_manager.py new file mode 100644 index 0000000..abd7dd1 --- /dev/null +++ b/gomill/gomill/job_manager.py @@ -0,0 +1,218 @@ +"""Job system supporting multiprocessing.""" + +import sys + +from gomill import compact_tracebacks + +multiprocessing = None + +NoJobAvailable = object() + +class JobFailed(StandardError): + """Error reported by a job.""" + +class JobSourceError(StandardError): + """Error from a job source object.""" + +class JobError(object): + """Error from a job.""" + def __init__(self, job, msg): + self.job = job + self.msg = msg + +def _initialise_multiprocessing(): + global multiprocessing + if multiprocessing is not None: + return + try: + import multiprocessing + except ImportError: + multiprocessing = None + +class Worker_finish_signal(object): + pass +worker_finish_signal = Worker_finish_signal() + +def worker_run_jobs(job_queue, response_queue): + try: + #pid = os.getpid() + #sys.stderr.write("worker %d starting\n" % pid) + while True: + job = job_queue.get() + #sys.stderr.write("worker %d: %s\n" % (pid, repr(job))) + if isinstance(job, Worker_finish_signal): + break + try: + response = job.run() + except JobFailed, e: + response = JobError(job, str(e)) + sys.exc_clear() + del e + except Exception: + response = JobError( + job, compact_tracebacks.format_traceback(skip=1)) + sys.exc_clear() + response_queue.put(response) + #sys.stderr.write("worker %d finishing\n" % pid) + response_queue.cancel_join_thread() + # Unfortunately, there will be places in the child that this doesn't cover. + # But it will avoid the ugly traceback in most cases. + except KeyboardInterrupt: + sys.exit(3) + +class Job_manager(object): + def __init__(self): + self.passed_exceptions = [] + + def pass_exception(self, cls): + self.passed_exceptions.append(cls) + +class Multiprocessing_job_manager(Job_manager): + def __init__(self, number_of_workers): + Job_manager.__init__(self) + _initialise_multiprocessing() + if multiprocessing is None: + raise StandardError("multiprocessing not available") + if not 1 <= number_of_workers < 1024: + raise ValueError + self.number_of_workers = number_of_workers + + def start_workers(self): + self.job_queue = multiprocessing.Queue() + self.response_queue = multiprocessing.Queue() + self.workers = [] + for i in range(self.number_of_workers): + worker = multiprocessing.Process( + target=worker_run_jobs, + args=(self.job_queue, self.response_queue)) + self.workers.append(worker) + for worker in self.workers: + worker.start() + + def run_jobs(self, job_source): + active_jobs = 0 + while True: + if active_jobs < self.number_of_workers: + try: + job = job_source.get_job() + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from get_job()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + if job is not NoJobAvailable: + #sys.stderr.write("MGR: sending %s\n" % repr(job)) + self.job_queue.put(job) + active_jobs += 1 + continue + if active_jobs == 0: + break + + response = self.response_queue.get() + if isinstance(response, JobError): + try: + job_source.process_error_response( + response.job, response.msg) + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from process_error_response()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + else: + try: + job_source.process_response(response) + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from process_response()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + active_jobs -= 1 + #sys.stderr.write("MGR: received response %s\n" % repr(response)) + + def finish(self): + for _ in range(self.number_of_workers): + self.job_queue.put(worker_finish_signal) + for worker in self.workers: + worker.join() + self.job_queue = None + self.response_queue = None + +class In_process_job_manager(Job_manager): + def start_workers(self): + pass + + def run_jobs(self, job_source): + while True: + try: + job = job_source.get_job() + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from get_job()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + if job is NoJobAvailable: + break + try: + response = job.run() + except Exception, e: + if isinstance(e, JobFailed): + msg = str(e) + else: + msg = compact_tracebacks.format_traceback(skip=1) + try: + job_source.process_error_response(job, msg) + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from process_error_response()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + else: + try: + job_source.process_response(response) + except Exception, e: + for cls in self.passed_exceptions: + if isinstance(e, cls): + raise + raise JobSourceError( + "error from process_response()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + + def finish(self): + pass + +def run_jobs(job_source, max_workers=None, allow_mp=True, + passed_exceptions=None): + if allow_mp: + _initialise_multiprocessing() + if multiprocessing is None: + allow_mp = False + if allow_mp: + if max_workers is None: + max_workers = multiprocessing.cpu_count() + job_manager = Multiprocessing_job_manager(max_workers) + else: + job_manager = In_process_job_manager() + if passed_exceptions: + for cls in passed_exceptions: + job_manager.pass_exception(cls) + job_manager.start_workers() + try: + job_manager.run_jobs(job_source) + except Exception: + try: + job_manager.finish() + except Exception, e2: + print >>sys.stderr, "Error closing down workers:\n%s" % e2 + raise + job_manager.finish() + diff --git a/gomill/gomill/mcts_tuners.py b/gomill/gomill/mcts_tuners.py new file mode 100644 index 0000000..dc1e4c9 --- /dev/null +++ b/gomill/gomill/mcts_tuners.py @@ -0,0 +1,848 @@ +"""Competitions for parameter tuning using Monte-carlo tree search.""" + +from __future__ import division + +import operator +import random +from heapq import nlargest +from math import exp, log, sqrt + +from gomill import compact_tracebacks +from gomill import game_jobs +from gomill import competitions +from gomill import competition_schedulers +from gomill.competitions import ( + Competition, NoGameAvailable, CompetitionError, ControlFileError, + Player_config) +from gomill.settings import * + + +class Node(object): + """A MCTS node. + + Public attributes: + children -- list of Nodes, or None for unexpanded + wins + visits + value -- wins / visits + rsqrt_visits -- 1 / sqrt(visits) + + """ + def count_tree_size(self): + if self.children is None: + return 1 + return sum(child.count_tree_size() for child in self.children) + 1 + + def recalculate(self): + """Update value and rsqrt_visits from changed wins and visits.""" + self.value = self.wins / self.visits + self.rsqrt_visits = sqrt(1/self.visits) + + def __getstate__(self): + return (self.children, self.wins, self.visits) + + def __setstate__(self, state): + self.children, self.wins, self.visits = state + self.recalculate() + + __slots__ = ( + 'children', + 'wins', + 'visits', + 'value', + 'rsqrt_visits', + ) + + def __repr__(self): + return "" % (self.value, repr(self.children)) + + +class Tree(object): + """A tree of MCTS nodes representing N-dimensional parameter space. + + Parameters (available as read-only attributes): + splits -- subdivisions of each dimension + (list of integers, one per dimension) + max_depth -- number of generations below the root + initial_visits -- visit count for newly-created nodes + initial_wins -- win count for newly-created nodes + exploration_coefficient -- constant for UCT formula (float) + + Public attributes: + root -- Node + dimensions -- number of dimensions in the parameter space + + All changing state is in the tree of Node objects started at 'root'. + + References to 'optimiser_parameters' below mean a sequence of length + 'dimensions', whose values are floats in the range 0.0..1.0 representing + a point in this space. + + Each node in the tree represents an N-cuboid of parameter space. Each + expanded node has prod(splits) children, tiling its cuboid. + + (The splits are the same in each generation.) + + Instantiate with: + all parameters listed above + parameter_formatter -- function optimiser_parameters -> string + + """ + def __init__(self, splits, max_depth, + exploration_coefficient, + initial_visits, initial_wins, + parameter_formatter): + self.splits = splits + self.dimensions = len(splits) + self.branching_factor = reduce(operator.mul, splits) + self.max_depth = max_depth + self.exploration_coefficient = exploration_coefficient + self.initial_visits = initial_visits + self.initial_wins = initial_wins + self._initial_value = initial_wins / initial_visits + self._initial_rsqrt_visits = 1/sqrt(initial_visits) + self.format_parameters = parameter_formatter + + # map child index -> coordinate vector + # coordinate vector -- tuple length 'dimensions' with values in + # range(splits[d]) + # The first dimension changes most slowly. + self._cube_coordinates = [] + for child_index in xrange(self.branching_factor): + v = [] + i = child_index + for split in reversed(splits): + i, coord = divmod(i, split) + v.append(coord) + v.reverse() + self._cube_coordinates.append(tuple(v)) + + def new_root(self): + """Initialise the tree with an expanded root node.""" + self.node_count = 1 # For description only + self.root = Node() + self.root.children = None + self.root.wins = self.initial_wins + self.root.visits = self.initial_visits + self.root.value = self.initial_wins / self.initial_visits + self.root.rsqrt_visits = self._initial_rsqrt_visits + self.expand(self.root) + + def set_root(self, node): + """Use the specified node as the tree's root. + + This is used when restoring serialised state. + + Raises ValueError if the node doesn't have the expected number of + children. + + """ + if not node.children or len(node.children) != self.branching_factor: + raise ValueError + self.root = node + self.node_count = node.count_tree_size() + + def expand(self, node): + """Add children to the specified node.""" + assert node.children is None + node.children = [] + child_count = self.branching_factor + for _ in xrange(child_count): + child = Node() + child.children = None + child.wins = self.initial_wins + child.visits = self.initial_visits + child.value = self._initial_value + child.rsqrt_visits = self._initial_rsqrt_visits + node.children.append(child) + self.node_count += child_count + + def is_ripe(self, node): + """Say whether a node has been visted enough times to be expanded.""" + return node.visits != self.initial_visits + + def parameters_for_path(self, choice_path): + """Retrieve the point in parameter space given by a node. + + choice_path -- sequence of child indices + + Returns optimiser_parameters representing the centre of the region + of parameter space represented by the node of interest. + + choice_path must represent a path from the root to the node of interest. + + """ + lo = [0.0] * self.dimensions + breadths = [1.0] * self.dimensions + for child_index in choice_path: + cube_pos = self._cube_coordinates[child_index] + breadths = [f/split for (f, split) in zip(breadths, self.splits)] + for d, coord in enumerate(cube_pos): + lo[d] += breadths[d] * coord + return [f + .5*breadth for (f, breadth) in zip(lo, breadths)] + + def retrieve_best_parameters(self): + """Find the parameters with the most promising simulation results. + + Returns optimiser_parameters + + This walks the tree from the root, at each point choosing the node with + most wins, and returns the parameters corresponding to the leaf node. + + """ + simulation = self.retrieve_best_parameter_simulation() + return simulation.get_parameters() + + def retrieve_best_parameter_simulation(self): + """Return the Greedy_simulation used for retrieve_best_parameters.""" + simulation = Greedy_simulation(self) + simulation.walk() + return simulation + + def get_test_parameters(self): + """Return a 'typical' optimiser_parameters.""" + return self.parameters_for_path([0]) + + def describe_choice(self, choice): + """Return a string describing a child's coordinates in its parent.""" + return str(self._cube_coordinates[choice]).replace(" ", "") + + def describe(self): + """Return a text description of the current state of the tree. + + This currently dumps the full tree to depth 2. + + """ + + def describe_node(node, choice_path): + parameters = self.format_parameters( + self.parameters_for_path(choice_path)) + choice_s = self.describe_choice(choice_path[-1]) + return "%s %s %.3f %3d" % ( + choice_s, parameters, node.value, + node.visits - self.initial_visits) + + root = self.root + wins = root.wins - self.initial_wins + visits = root.visits - self.initial_visits + try: + win_rate = "%.3f" % (wins/visits) + except ZeroDivisionError: + win_rate = "--" + result = [ + "%d nodes" % self.node_count, + "Win rate %d/%d = %s" % (wins, visits, win_rate) + ] + + for choice, node in enumerate(self.root.children): + result.append(" " + describe_node(node, [choice])) + if node.children is None: + continue + for choice2, node2 in enumerate(node.children): + result.append(" " + describe_node(node2, [choice, choice2])) + return "\n".join(result) + + def summarise(self, out, summary_spec): + """Write a summary of the most-visited parts of the tree. + + out -- writeable file-like object + summary_spec -- list of ints + + summary_spec says how many nodes to describe at each depth of the tree + (so to show only direct children of the root, pass a list of length 1). + + """ + def p(s): + print >>out, s + + def describe_node(node, choice_path): + parameters = self.format_parameters( + self.parameters_for_path(choice_path)) + choice_s = " ".join(map(self.describe_choice, choice_path)) + return "%s %-40s %.3f %3d" % ( + choice_s, parameters, node.value, + node.visits - self.initial_visits) + + def most_visits((child_index, node)): + return node.visits + + last_generation = [([], self.root)] + for i, n in enumerate(summary_spec): + depth = i + 1 + p("most visited at depth %s" % (depth)) + + this_generation = [] + for path, node in last_generation: + if node.children is not None: + this_generation += [ + (path + [child_index], child) + for (child_index, child) in enumerate(node.children)] + + for path, node in sorted( + nlargest(n, this_generation, key=most_visits)): + p(describe_node(node, path)) + last_generation = this_generation + p("") + + +class Simulation(object): + """A single monte-carlo simulation. + + Instantiate with the Tree the simulation will run in. + + Use the methods in the following order: + run() + get_parameters() + update_stats(b) + describe() + + """ + def __init__(self, tree): + self.tree = tree + # list of Nodes + self.node_path = [] + # corresponding list of child indices + self.choice_path = [] + # bool + self.candidate_won = None + + def _choose_action(self, node): + """Choose the best action from the specified node. + + Returns a pair (child index, node) + + """ + uct_numerator = (self.tree.exploration_coefficient * + sqrt(log(node.visits))) + def urgency((i, child)): + return child.value + uct_numerator * child.rsqrt_visits + start = random.randrange(len(node.children)) + children = list(enumerate(node.children)) + return max(children[start:] + children[:start], key=urgency) + + def walk(self): + """Choose a node sequence, without expansion.""" + node = self.tree.root + while node.children is not None: + choice, node = self._choose_action(node) + self.node_path.append(node) + self.choice_path.append(choice) + + def run(self): + """Choose the node sequence for this simulation. + + This walks down from the root, using _choose_action() at each level, + until it reaches a leaf; if the leaf has already been visited, this + expands it and chooses one more action. + + """ + self.walk() + node = self.node_path[-1] + if (len(self.node_path) < self.tree.max_depth and + self.tree.is_ripe(node)): + self.tree.expand(node) + choice, child = self._choose_action(node) + self.node_path.append(child) + self.choice_path.append(choice) + + def get_parameters(self): + """Retrieve the parameters corresponding to the simulation's leaf node. + + Returns optimiser_parameters + + """ + return self.tree.parameters_for_path(self.choice_path) + + def update_stats(self, candidate_won): + """Update the tree's node statistics with the simulation's results. + + This updates visits (and wins, if appropriate) for each node in the + simulation's node sequence. + + """ + self.candidate_won = candidate_won + for node in self.node_path: + node.visits += 1 + if candidate_won: + node.wins += 1 + node.recalculate() + self.tree.root.visits += 1 + if candidate_won: + self.tree.root.wins += 1 # For description only + self.tree.root.recalculate() + + def describe_steps(self): + """Return a text description of the simulation's node sequence.""" + return " ".join(map(self.tree.describe_choice, self.choice_path)) + + def describe(self): + """Return a one-line-ish text description of the simulation.""" + result = "%s [%s]" % ( + self.tree.format_parameters(self.get_parameters()), + self.describe_steps()) + if self.candidate_won is not None: + result += (" lost", " won")[self.candidate_won] + return result + + def describe_briefly(self): + """Return a shorter description of the simulation.""" + return "%s %s" % (self.tree.format_parameters(self.get_parameters()), + ("lost", "won")[self.candidate_won]) + +class Greedy_simulation(Simulation): + """Variant of simulation that chooses the node with most wins. + + This is used to pick the 'best' parameters from the current state of the + tree. + + """ + def _choose_action(self, node): + def wins((i, node)): + return node.wins + return max(enumerate(node.children), key=wins) + + +parameter_settings = [ + Setting('code', interpret_identifier), + Setting('scale', interpret_callable), + Setting('split', interpret_positive_int), + Setting('format', interpret_8bit_string, default=None), + ] + +class Parameter_config(Quiet_config): + """Parameter (ie, dimension) description for use in control files.""" + # positional or keyword + positional_arguments = ('code',) + # keyword-only + keyword_arguments = tuple(setting.name for setting in parameter_settings + if setting.name != 'code') + +class Parameter_spec(object): + """Internal description of a parameter spec from the configuration file. + + Public attributes: + code -- identifier + split -- integer + scale -- function float(0.0..1.0) -> player parameter + format -- string for use with '%' + + """ + +class Scale_fn(object): + """Callable implementing a scale function. + + Scale_fn classes are used to provide a convenient way to describe scale + functions in the control file (LINEAR, LOG, ...). + + """ + +class Linear_scale_fn(Scale_fn): + """Linear scale function. + + Instantiate with + lower_bound -- float + upper_bound -- float + integer -- bool (means 'round result to nearest integer') + + """ + def __init__(self, lower_bound, upper_bound, integer=False): + self.lower_bound = float(lower_bound) + self.upper_bound = float(upper_bound) + self.range = float(upper_bound - lower_bound) + self.integer = bool(integer) + + def __call__(self, f): + result = (f * self.range) + self.lower_bound + if self.integer: + result = int(result+.5) + return result + +class Log_scale_fn(Scale_fn): + """Log scale function. + + Instantiate with + lower_bound -- float + upper_bound -- float + integer -- bool (means 'round result to nearest integer') + + """ + def __init__(self, lower_bound, upper_bound, integer=False): + if lower_bound == 0.0: + raise ValueError("lower bound is zero") + self.rate = log(upper_bound / lower_bound) + self.lower_bound = lower_bound + self.integer = bool(integer) + + def __call__(self, f): + result = exp(self.rate*f) * self.lower_bound + if self.integer: + result = int(result+.5) + return result + +class Explicit_scale_fn(Scale_fn): + """Scale function that returns elements from a list. + + Instantiate with the list of values to use. + + Normally use this with 'split' equal to the length of the list + (more generally, split**max_depth equal to the length of the list). + + """ + def __init__(self, values): + if not values: + raise ValueError("empty value list") + self.values = tuple(values) + self.n = len(values) + + def __call__(self, f): + return self.values[int(self.n * f)] + + +class LINEAR(Config_proxy): + underlying = Linear_scale_fn + +class LOG(Config_proxy): + underlying = Log_scale_fn + +class EXPLICIT(Config_proxy): + underlying = Explicit_scale_fn + + +class Mcts_tuner(Competition): + """A Competition for parameter tuning using the Monte-carlo tree search. + + The game ids are strings containing integers starting from zero. + + """ + def __init__(self, competition_code, **kwargs): + Competition.__init__(self, competition_code, **kwargs) + self.outstanding_simulations = {} + self.halt_on_next_failure = True + + def control_file_globals(self): + result = Competition.control_file_globals(self) + result.update({ + 'Parameter' : Parameter_config, + 'LINEAR' : LINEAR, + 'LOG' : LOG, + 'EXPLICIT' : EXPLICIT, + }) + return result + + global_settings = (Competition.global_settings + + competitions.game_settings + [ + Setting('number_of_games', allow_none(interpret_int), default=None), + Setting('candidate_colour', interpret_colour), + Setting('log_tree_to_history_period', + allow_none(interpret_positive_int), default=None), + Setting('summary_spec', interpret_sequence_of(interpret_int), + default=(30,)), + Setting('number_of_running_simulations_to_show', interpret_int, + default=12), + ]) + + special_settings = [ + Setting('opponent', interpret_identifier), + Setting('parameters', + interpret_sequence_of_quiet_configs(Parameter_config)), + Setting('make_candidate', interpret_callable), + ] + + # These are used to instantiate Tree; they don't turn into Mcts_tuner + # attributes. + tree_settings = [ + Setting('max_depth', interpret_positive_int, default=1), + Setting('exploration_coefficient', interpret_float), + Setting('initial_visits', interpret_positive_int), + Setting('initial_wins', interpret_positive_int), + ] + + def parameter_spec_from_config(self, parameter_config): + """Make a Parameter_spec from a Parameter_config. + + Raises ControlFileError if there is an error in the configuration. + + Returns a Parameter_spec with all attributes set. + + """ + arguments = parameter_config.resolve_arguments() + interpreted = load_settings(parameter_settings, arguments) + pspec = Parameter_spec() + for name, value in interpreted.iteritems(): + setattr(pspec, name, value) + optimiser_param = 1.0/(pspec.split*2) + try: + scaled = pspec.scale(optimiser_param) + except Exception: + raise ValueError( + "error from scale (applied to %s)\n%s" % + (optimiser_param, compact_tracebacks.format_traceback(skip=1))) + if pspec.format is None: + pspec.format = pspec.code + ":%s" + try: + pspec.format % scaled + except Exception: + raise ControlFileError("'format': invalid format string") + return pspec + + def initialise_from_control_file(self, config): + Competition.initialise_from_control_file(self, config) + + if self.komi == int(self.komi): + raise ControlFileError("komi: must be fractional to prevent jigos") + + competitions.validate_handicap( + self.handicap, self.handicap_style, self.board_size) + + try: + specials = load_settings(self.special_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + + try: + self.opponent = self.players[specials['opponent']] + except KeyError: + raise ControlFileError( + "opponent: unknown player %s" % specials['opponent']) + + self.parameter_specs = [] + if not specials['parameters']: + raise ControlFileError("parameters: empty list") + seen_codes = set() + for i, parameter_spec in enumerate(specials['parameters']): + try: + pspec = self.parameter_spec_from_config(parameter_spec) + except StandardError, e: + code = parameter_spec.get_key() + if code is None: + code = i + raise ControlFileError("parameter %s: %s" % (code, e)) + if pspec.code in seen_codes: + raise ControlFileError( + "duplicate parameter code: %s" % pspec.code) + seen_codes.add(pspec.code) + self.parameter_specs.append(pspec) + + self.candidate_maker_fn = specials['make_candidate'] + + try: + tree_arguments = load_settings(self.tree_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + self.tree = Tree(splits=[pspec.split for pspec in self.parameter_specs], + parameter_formatter=self.format_optimiser_parameters, + **tree_arguments) + + + # State attributes (*: in persistent state): + # *scheduler -- Simple_scheduler + # *tree -- Tree (root node is persisted) + # outstanding_simulations -- map game_number -> Simulation + # halt_on_next_failure -- bool + # *opponent_description -- string (or None) + + def set_clean_status(self): + self.scheduler = competition_schedulers.Simple_scheduler() + self.tree.new_root() + self.opponent_description = None + + # Can bump this to prevent people loading incompatible .status files. + status_format_version = 0 + + def get_status(self): + # path0 is stored for consistency check + return { + 'scheduler' : self.scheduler, + 'tree_root' : self.tree.root, + 'opponent_description' : self.opponent_description, + 'path0' : self.scale_parameters(self.tree.parameters_for_path([0])), + } + + def set_status(self, status): + root = status['tree_root'] + try: + self.tree.set_root(root) + except ValueError: + raise CompetitionError( + "status file is inconsistent with control file") + expected_path0 = self.scale_parameters( + self.tree.parameters_for_path([0])) + if status['path0'] != expected_path0: + raise CompetitionError( + "status file is inconsistent with control file") + self.scheduler = status['scheduler'] + self.scheduler.rollback() + self.opponent_description = status['opponent_description'] + + def scale_parameters(self, optimiser_parameters): + l = [] + for pspec, v in zip(self.parameter_specs, optimiser_parameters): + try: + l.append(pspec.scale(v)) + except Exception: + raise CompetitionError( + "error from scale for %s\n%s" % + (pspec.code, compact_tracebacks.format_traceback(skip=1))) + return tuple(l) + + def format_engine_parameters(self, engine_parameters): + l = [] + for pspec, v in zip(self.parameter_specs, engine_parameters): + try: + s = pspec.format % v + except Exception: + s = "[%s?%s]" % (pspec.code, v) + l.append(s) + return "; ".join(l) + + def format_optimiser_parameters(self, optimiser_parameters): + return self.format_engine_parameters(self.scale_parameters( + optimiser_parameters)) + + def make_candidate(self, player_code, engine_parameters): + """Make a player using the specified engine parameters. + + Returns a game_jobs.Player. + + """ + try: + candidate_config = self.candidate_maker_fn(*engine_parameters) + except Exception: + raise CompetitionError( + "error from make_candidate()\n%s" % + compact_tracebacks.format_traceback(skip=1)) + if not isinstance(candidate_config, Player_config): + raise CompetitionError( + "make_candidate() returned %r, not Player" % + candidate_config) + try: + candidate = self.game_jobs_player_from_config( + player_code, candidate_config) + except Exception, e: + raise CompetitionError( + "bad player spec from make_candidate():\n" + "%s\nparameters were: %s" % + (e, self.format_engine_parameters(engine_parameters))) + return candidate + + def get_player_checks(self): + test_parameters = self.tree.get_test_parameters() + engine_parameters = self.scale_parameters(test_parameters) + candidate = self.make_candidate('candidate', engine_parameters) + result = [] + for player in [candidate, self.opponent]: + check = game_jobs.Player_check() + check.player = player + check.board_size = self.board_size + check.komi = self.komi + result.append(check) + return result + + def get_game(self): + if (self.number_of_games is not None and + self.scheduler.issued >= self.number_of_games): + return NoGameAvailable + game_number = self.scheduler.issue() + + simulation = Simulation(self.tree) + simulation.run() + optimiser_parameters = simulation.get_parameters() + engine_parameters = self.scale_parameters(optimiser_parameters) + candidate = self.make_candidate("#%d" % game_number, engine_parameters) + self.outstanding_simulations[game_number] = simulation + + job = game_jobs.Game_job() + job.game_id = str(game_number) + job.game_data = game_number + if self.candidate_colour == 'b': + job.player_b = candidate + job.player_w = self.opponent + else: + job.player_b = self.opponent + job.player_w = candidate + job.board_size = self.board_size + job.komi = self.komi + job.move_limit = self.move_limit + job.handicap = self.handicap + job.handicap_is_free = (self.handicap_style == 'free') + job.use_internal_scorer = (self.scorer == 'internal') + job.internal_scorer_handicap_compensation = \ + self.internal_scorer_handicap_compensation + job.sgf_event = self.competition_code + job.sgf_note = ("Candidate parameters: %s" % + self.format_engine_parameters(engine_parameters)) + return job + + def process_game_result(self, response): + self.halt_on_next_failure = False + self.opponent_description = response.engine_descriptions[ + self.opponent.code] + game_number = response.game_data + self.scheduler.fix(game_number) + # Counting no-result as loss for the candidate + candidate_won = ( + response.game_result.winning_colour == self.candidate_colour) + simulation = self.outstanding_simulations.pop(game_number) + simulation.update_stats(candidate_won) + self.log_history(simulation.describe()) + if (self.log_tree_to_history_period is not None and + self.scheduler.fixed % self.log_tree_to_history_period == 0): + self.log_history(self.tree.describe()) + return "%s %s" % (simulation.describe(), + response.game_result.sgf_result) + + def process_game_error(self, job, previous_error_count): + ## If the very first game to return a response gives an error, halt. + ## If two games in a row give an error, halt. + ## Otherwise, forget about the failed game + stop_competition = False + retry_game = False + game_number = job.game_data + del self.outstanding_simulations[game_number] + self.scheduler.fix(game_number) + if self.halt_on_next_failure: + stop_competition = True + else: + self.halt_on_next_failure = True + return stop_competition, retry_game + + def write_static_description(self, out): + def p(s): + print >>out, s + p("MCTS tuning event: %s" % self.competition_code) + if self.description: + p(self.description) + p("board size: %s" % self.board_size) + p("komi: %s" % self.komi) + + def _write_main_report(self, out): + games_played = self.scheduler.fixed + if self.number_of_games is None: + print >>out, "%d games played" % games_played + else: + print >>out, "%d/%d games played" % ( + games_played, self.number_of_games) + print >>out + best_simulation = self.tree.retrieve_best_parameter_simulation() + print >>out, "Best parameters: %s" % best_simulation.describe() + print >>out + self.tree.summarise(out, self.summary_spec) + + def write_screen_report(self, out): + self._write_main_report(out) + if self.outstanding_simulations: + print >>out, "In progress:" + to_show = sorted(self.outstanding_simulations.iteritems())\ + [:self.number_of_running_simulations_to_show] + for game_id, simulation in to_show: + print >>out, "game %s: %s" % (game_id, simulation.describe()) + + def write_short_report(self, out): + self.write_static_description(out) + self._write_main_report(out) + if self.opponent_description: + print >>out, "opponent: %s" % self.opponent_description + print >>out + + write_full_report = write_short_report + diff --git a/gomill/gomill/playoffs.py b/gomill/gomill/playoffs.py new file mode 100644 index 0000000..bfbbd82 --- /dev/null +++ b/gomill/gomill/playoffs.py @@ -0,0 +1,179 @@ +"""Competitions made up of repeated matchups between specified players.""" + +from gomill import game_jobs +from gomill import competitions +from gomill import tournaments +from gomill.competitions import (Competition, ControlFileError) +from gomill.settings import * + + +class Matchup_config(Quiet_config): + """Matchup description for use in control files.""" + # positional or keyword + positional_arguments = ('player_1', 'player_2') + # keyword-only + keyword_arguments = ( + ('id', 'name') + + tuple(setting.name for setting in tournaments.matchup_settings)) + + +class Playoff(tournaments.Tournament): + """A Tournament with explicitly listed matchups. + + The game ids are like '0_2', where 0 is the matchup id and 2 is the game + number within the matchup. + + """ + + def control_file_globals(self): + result = Competition.control_file_globals(self) + result.update({ + 'Matchup' : Matchup_config, + }) + return result + + + special_settings = [ + Setting('matchups', + interpret_sequence_of_quiet_configs(Matchup_config)), + ] + + def matchup_from_config(self, matchup_number, + matchup_config, matchup_defaults): + """Make a Matchup from a Matchup_config. + + This does the following checks and fixups before calling make_matchup(): + + Checks that the player_1 and player_2 parameters exist, and that the + player codes are present in self.players. + + Validates all the matchup_config arguments, and merges them with the + defaults. + + If player_1 and player_2 are the same, takes the following actions: + - sets player_2 to #2 + - if it doesn't already exist, creates #2 as a clone of + player_1 and adds it to self.players + + """ + matchup_id = str(matchup_number) + try: + arguments = matchup_config.resolve_arguments() + if 'id' in arguments: + try: + matchup_id = interpret_identifier(arguments['id']) + except ValueError, e: + raise ValueError("'id': %s" % e) + try: + player_1 = arguments['player_1'] + player_2 = arguments['player_2'] + except KeyError: + raise ControlFileError("not enough arguments") + if player_1 not in self.players: + raise ControlFileError("unknown player %s" % player_1) + if player_2 not in self.players: + raise ControlFileError("unknown player %s" % player_2) + # If both players are the same, make a clone. + if player_1 == player_2: + player_2 += "#2" + if player_2 not in self.players: + self.players[player_2] = \ + self.players[player_1].copy(player_2) + interpreted = load_settings( + tournaments.matchup_settings, arguments, + apply_defaults=False, allow_missing=True) + matchup_name = arguments.get('name') + if matchup_name is not None: + try: + matchup_name = interpret_as_utf8(matchup_name) + except ValueError, e: + raise ValueError("'name': %s" % e) + parameters = matchup_defaults.copy() + parameters.update(interpreted) + return self.make_matchup( + matchup_id, player_1, player_2, + parameters, matchup_name) + except StandardError, e: + raise ControlFileError("matchup %s: %s" % (matchup_id, e)) + + + def initialise_from_control_file(self, config): + Competition.initialise_from_control_file(self, config) + + try: + matchup_defaults = load_settings( + tournaments.matchup_settings, config, allow_missing=True) + except ValueError, e: + raise ControlFileError(str(e)) + + # Check default handicap settings when possible, for friendlier error + # reporting (would be caught in the matchup anyway). + if 'board_size' in matchup_defaults: + try: + competitions.validate_handicap( + matchup_defaults['handicap'], + matchup_defaults['handicap_style'], + matchup_defaults['board_size']) + except ControlFileError, e: + raise ControlFileError("default %s" % e) + + try: + specials = load_settings(self.special_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + + # map matchup_id -> Matchup + self.matchups = {} + # Matchups in order of definition + self.matchup_list = [] + if not specials['matchups']: + raise ControlFileError("matchups: empty list") + + for i, matchup_config in enumerate(specials['matchups']): + m = self.matchup_from_config(i, matchup_config, matchup_defaults) + if m.id in self.matchups: + raise ControlFileError("duplicate matchup id '%s'" % m.id) + self.matchups[m.id] = m + self.matchup_list.append(m) + + + # Can bump this to prevent people loading incompatible .status files. + status_format_version = 1 + + def get_player_checks(self): + # For board size and komi, we check the values from the first matchup + # the player appears in. + used_players = {} + for m in reversed(self.matchup_list): + if m.number_of_games == 0: + continue + used_players[m.player_1] = m + used_players[m.player_2] = m + result = [] + for code, matchup in sorted(used_players.iteritems()): + check = game_jobs.Player_check() + check.player = self.players[code] + check.board_size = matchup.board_size + check.komi = matchup.komi + result.append(check) + return result + + + def write_screen_report(self, out): + self.write_matchup_reports(out) + + def write_short_report(self, out): + def p(s): + print >>out, s + p("playoff: %s" % self.competition_code) + if self.description: + p(self.description) + p('') + self.write_screen_report(out) + self.write_ghost_matchup_reports(out) + p('') + self.write_player_descriptions(out) + p('') + + write_full_report = write_short_report + diff --git a/gomill/gomill/ringmaster_command_line.py b/gomill/gomill/ringmaster_command_line.py new file mode 100644 index 0000000..d963e54 --- /dev/null +++ b/gomill/gomill/ringmaster_command_line.py @@ -0,0 +1,122 @@ +"""Command-line interface to the ringmaster.""" + +import os +import sys +from optparse import OptionParser + +from gomill import compact_tracebacks +from gomill.ringmasters import ( + Ringmaster, RingmasterError, RingmasterInternalError) + + +# Action functions return the desired exit status; implicit return is fine to +# indicate a successful exit. + +def do_run(ringmaster, options): + if not options.quiet: + print "running startup checks on all players" + if not ringmaster.check_players(discard_stderr=True): + print "(use the 'check' command to see stderr output)" + return 1 + if options.log_gtp: + ringmaster.enable_gtp_logging() + if options.quiet: + ringmaster.set_display_mode('quiet') + if ringmaster.status_file_exists(): + ringmaster.load_status() + else: + ringmaster.set_clean_status() + if options.parallel is not None: + ringmaster.set_parallel_worker_count(options.parallel) + ringmaster.run(options.max_games) + ringmaster.report() + +def do_stop(ringmaster, options): + ringmaster.write_command("stop") + +def do_show(ringmaster, options): + if not ringmaster.status_file_exists(): + raise RingmasterError("no status file") + ringmaster.load_status() + ringmaster.print_status_report() + +def do_report(ringmaster, options): + if not ringmaster.status_file_exists(): + raise RingmasterError("no status file") + ringmaster.load_status() + ringmaster.report() + +def do_reset(ringmaster, options): + ringmaster.delete_state_and_output() + +def do_check(ringmaster, options): + if not ringmaster.check_players(discard_stderr=False): + return 1 + +def do_debugstatus(ringmaster, options): + ringmaster.print_status() + +_actions = { + "run" : do_run, + "stop" : do_stop, + "show" : do_show, + "report" : do_report, + "reset" : do_reset, + "check" : do_check, + "debugstatus" : do_debugstatus, + } + + +def run(argv, ringmaster_class): + usage = ("%prog [options] [command]\n\n" + "commands: run (default), stop, show, report, reset, check") + parser = OptionParser(usage=usage, prog="ringmaster", + version=ringmaster_class.public_version) + parser.add_option("--max-games", "-g", type="int", + help="maximum number of games to play in this run") + parser.add_option("--parallel", "-j", type="int", + help="number of worker processes") + parser.add_option("--quiet", "-q", action="store_true", + help="be silent except for warnings and errors") + parser.add_option("--log-gtp", action="store_true", + help="write GTP logs") + (options, args) = parser.parse_args(argv) + if len(args) == 0: + parser.error("no control file specified") + if len(args) > 2: + parser.error("too many arguments") + if len(args) == 1: + command = "run" + else: + command = args[1] + try: + action = _actions[command] + except KeyError: + parser.error("no such command: %s" % command) + ctl_pathname = args[0] + try: + if not os.path.exists(ctl_pathname): + raise RingmasterError("control file %s not found" % ctl_pathname) + ringmaster = ringmaster_class(ctl_pathname) + exit_status = action(ringmaster, options) + except RingmasterError, e: + print >>sys.stderr, "ringmaster:", e + exit_status = 1 + except KeyboardInterrupt: + exit_status = 3 + except RingmasterInternalError, e: + print >>sys.stderr, "ringmaster: internal error" + print >>sys.stderr, e + exit_status = 4 + except: + print >>sys.stderr, "ringmaster: internal error" + compact_tracebacks.log_traceback() + exit_status = 4 + sys.exit(exit_status) + +def main(): + run(sys.argv[1:], Ringmaster) + +if __name__ == "__main__": + main() + diff --git a/gomill/gomill/ringmaster_presenters.py b/gomill/gomill/ringmaster_presenters.py new file mode 100644 index 0000000..4d0aa24 --- /dev/null +++ b/gomill/gomill/ringmaster_presenters.py @@ -0,0 +1,180 @@ +"""Live display for ringmasters.""" + +import os +import subprocess +import sys +from cStringIO import StringIO + + +class Presenter(object): + """Abstract base class for presenters. + + This accepts messages on four _channels_, with codes + warnings + status + screen_report + results + + Warnings are always displayed immediately. + + Some presenters will delay display of other channels until refresh() is + called; some will display them immediately. + + """ + + # If this is true, ringmaster needn't bother doing the work to prepare most + # of the display. + shows_warnings_only = False + + def clear(self, channel): + """Clear the contents of the specified channel.""" + raise NotImplementedError + + def say(self, channel, s): + """Add a message to the specified channel. + + channel -- channel code + s -- string to display (no trailing newline) + + """ + raise NotImplementedError + + def refresh(self): + """Re-render the current screen. + + This typically displays the full status and screen_report, and the most + recent warnings and results. + + """ + raise NotImplementedError + + def get_stream(self, channel): + """Return a file-like object wired up to the specified channel. + + When the object is closed, the text written it is sent to the channel + (except that any trailing newline is removed). + + """ + return _Channel_writer(self, channel) + +class _Channel_writer(object): + """Support for get_stream() implementation.""" + def __init__(self, parent, channel): + self.parent = parent + self.channel = channel + self.stringio = StringIO() + + def write(self, s): + self.stringio.write(s) + + def close(self): + s = self.stringio.getvalue() + if s.endswith("\n"): + s = s[:-1] + self.parent.say(self.channel, s) + self.stringio.close() + + +class Quiet_presenter(Presenter): + """Presenter which shows only warnings. + + Warnings go to stderr. + + """ + shows_warnings_only = True + + def clear(self, channel): + pass + + def say(self, channel, s): + if channel == 'warnings': + print >>sys.stderr, s + + def refresh(self): + pass + + +class Box(object): + """Description of screen layout for the clearing presenter.""" + def __init__(self, name, heading, limit): + self.name = name + self.heading = heading + self.limit = limit + self.contents = [] + + def layout(self): + return "\n".join(self.contents[-self.limit:]) + +class Clearing_presenter(Presenter): + """Low-tech full-screen presenter. + + This shows all channels. + + """ + shows_warnings_only = False + + # warnings has to be last, so we can add to it immediately + box_specs = ( + ('status', None, 999), + ('screen_report', None, 999), + ('results', "Results", 6), + ('warnings', "Warnings", 4), + ) + + def __init__(self): + self.boxes = {} + self.box_list = [] + for t in self.box_specs: + box = Box(*t) + self.boxes[box.name] = box + self.box_list.append(box) + self.clear_method = None + + def clear(self, channel): + self.boxes[channel].contents = [] + + def say(self, channel, s): + self.boxes[channel].contents.append(s) + # 'warnings' box heading might be missing, but never mind. + if channel == 'warnings': + print s + + def refresh(self): + self.clear_screen() + for box in self.box_list: + if not box.contents: + continue + if box.heading: + print "= %s = " % box.heading + print box.layout() + if box.name != 'warnings': + print + + def screen_height(self): + """Return the current terminal height, or best guess.""" + return os.environ.get("LINES", 80) + + def clear_screen(self): + """Try to clear the terminal screen (if stdout is a terminal).""" + if self.clear_method is None: + try: + isatty = os.isatty(sys.stdout.fileno()) + except Exception: + isatty = False + if isatty: + self.clear_method = "clear" + else: + self.clear_method = "delimiter" + + if self.clear_method == "clear": + try: + retcode = subprocess.call("clear") + except Exception: + retcode = 1 + if retcode != 0: + self.clear_method = "newlines" + if self.clear_method == "newlines": + print "\n" * (self.screen_height()+1) + elif self.clear_method == "delimiter": + print 78 * "-" + diff --git a/gomill/gomill/ringmasters.py b/gomill/gomill/ringmasters.py new file mode 100644 index 0000000..176e6da --- /dev/null +++ b/gomill/gomill/ringmasters.py @@ -0,0 +1,769 @@ +"""Run competitions using GTP.""" + +from __future__ import division, with_statement + +import cPickle as pickle +import datetime +import errno +import os +import re +import shutil +import sys + +try: + import fcntl +except ImportError: + fcntl = None + +from gomill import compact_tracebacks +from gomill import game_jobs +from gomill import job_manager +from gomill import ringmaster_presenters +from gomill import terminal_input +from gomill.settings import * +from gomill.competitions import ( + NoGameAvailable, CompetitionError, ControlFileError) + +def interpret_python(source, provided_globals, display_filename): + """Interpret Python code from a unicode string. + + source -- unicode object + provided_globals -- dict + display_filename -- filename to use in exceptions + + The string is executed with a copy of provided_globals as the global and + local namespace. Returns that namespace. + + The source string must not have an encoding declaration (SyntaxError will be + raised if it does). + + Propagates exceptions. + + """ + result = provided_globals.copy() + code = compile(source, display_filename, 'exec', + division.compiler_flag, True) + exec code in result + return result + +class RingmasterError(StandardError): + """Error reported by a Ringmaster.""" + +class RingmasterInternalError(StandardError): + """Error reported by a Ringmaster which indicates a bug.""" + + +class Ringmaster(object): + """Manage a competition as described by a control file. + + Most methods can raise RingmasterError. + + Instantiate with the pathname of the control file. The control file is read + and interpreted at instantiation time (and errors are reported at that + point). + + Ringmaster objects are used as a job source for the job manager. + + """ + # Can bump this to prevent people loading incompatible .status files. + status_format_version = 0 + + # For --version command + public_version = "gomill ringmaster v0.7.2" + + def __init__(self, control_pathname): + """Instantiate and initialise a Ringmaster. + + Reads the control file. + + Creates the Competition and initialises it from the control file. + + """ + self.display_mode = 'clearing' + self.worker_count = None + self.max_games_this_run = None + self.presenter = None + self.terminal_reader = None + self.stopping = False + self.stopping_reason = None + # Map game_id -> int + self.game_error_counts = {} + self.write_gtp_logs = False + + self.control_pathname = control_pathname + self.base_directory, control_filename = os.path.split(control_pathname) + self.competition_code, ext = os.path.splitext(control_filename) + if ext in (".log", ".status", ".cmd", ".hist", + ".report", ".games", ".void", ".gtplogs"): + raise RingmasterError("forbidden control file extension: %s" % ext) + stem = os.path.join(self.base_directory, self.competition_code) + self.log_pathname = stem + ".log" + self.status_pathname = stem + ".status" + self.command_pathname = stem + ".cmd" + self.history_pathname = stem + ".hist" + self.report_pathname = stem + ".report" + self.sgf_dir_pathname = stem + ".games" + self.void_dir_pathname = stem + ".void" + self.gtplog_dir_pathname = stem + ".gtplogs" + + self.status_is_loaded = False + try: + self._load_control_file() + except ControlFileError, e: + raise RingmasterError("error in control file:\n%s" % e) + + + def _read_control_file(self): + """Return the contents of the control file as an 8-bit string.""" + try: + with open(self.control_pathname) as f: + return f.read() + except EnvironmentError, e: + raise RingmasterError("failed to read control file:\n%s" % e) + + def _load_control_file(self): + """Main implementation for __init__.""" + + control_s = self._read_control_file() + + try: + self.competition_type = self._parse_competition_type(control_s) + except ValueError, e: + raise ControlFileError("can't find competition_type") + + try: + competition_class = self._get_competition_class( + self.competition_type) + except ValueError: + raise ControlFileError( + "unknown competition type: %s" % self.competition_type) + self.competition = competition_class(self.competition_code) + self.competition.set_base_directory(self.base_directory) + + try: + control_u = control_s.decode("utf-8") + except UnicodeDecodeError: + raise ControlFileError("file is not encoded in utf-8") + + try: + config = interpret_python( + control_u, self.competition.control_file_globals(), + display_filename=self.control_pathname) + except KeyboardInterrupt: + raise + except ControlFileError, e: + raise + except: + raise ControlFileError(compact_tracebacks.format_error_and_line()) + + if config.get("competition_type") != self.competition_type: + raise ControlFileError("competition_type improperly specified") + + try: + self._initialise_from_control_file(config) + except ControlFileError: + raise + except Exception, e: + raise RingmasterError("unhandled error in control file:\n%s" % + compact_tracebacks.format_traceback(skip=1)) + + try: + self.competition.initialise_from_control_file(config) + except ControlFileError: + raise + except Exception, e: + raise RingmasterError("unhandled error in control file:\n%s" % + compact_tracebacks.format_traceback(skip=1)) + + @staticmethod + def _parse_competition_type(source): + """Find the compitition_type definition in the control file. + + source -- string + + Requires the competition_type line to be the first non-comment line, and + to be a simple assignment of a string literal. + + Raises ValueError if it can't find the competition_type line, or if the + value isn't 'identifier-like'. + + """ + for line in source.split("\n"): + s = line.lstrip() + if not s or s.startswith("#"): + continue + break + else: + raise ValueError + # May propagate ValueError + m = re.match(r"competition_type\s*=\s*(['\"])([_a-zA-Z0-9]+)(['\"])$", + line) + if not m: + raise ValueError + if m.group(1) != m.group(3): + raise ValueError + return m.group(2) + + + @staticmethod + def _get_competition_class(competition_type): + """Find the competition class. + + competition_type -- string + + Returns a Competition subclass. + + Raises ValueError if the competition type is unknown. + + """ + if competition_type == "playoff": + from gomill import playoffs + return playoffs.Playoff + elif competition_type == "allplayall": + from gomill import allplayalls + return allplayalls.Allplayall + elif competition_type == "ce_tuner": + from gomill import cem_tuners + return cem_tuners.Cem_tuner + elif competition_type == "mc_tuner": + from gomill import mcts_tuners + return mcts_tuners.Mcts_tuner + else: + raise ValueError + + def _open_files(self): + """Open the log files and ensure that output directories exist. + + If flock is available, this takes out an exclusive lock on the log file. + If this lock is unavailable, it raises RingmasterError. + + Also removes the command file if it exists. + + """ + try: + self.logfile = open(self.log_pathname, "a") + except EnvironmentError, e: + raise RingmasterError("failed to open log file:\n%s" % e) + + if fcntl is not None: + try: + fcntl.flock(self.logfile, fcntl.LOCK_EX|fcntl.LOCK_NB) + except IOError, e: + if e.errno in (errno.EACCES, errno.EAGAIN): + raise RingmasterError("competition is already running") + except Exception: + pass + + try: + if os.path.exists(self.command_pathname): + os.remove(self.command_pathname) + except EnvironmentError, e: + raise RingmasterError("error removing existing .cmd file:\n%s" % e) + + try: + self.historyfile = open(self.history_pathname, "a") + except EnvironmentError, e: + raise RingmasterError("failed to open history file:\n%s" % e) + + if self.record_games: + try: + if not os.path.exists(self.sgf_dir_pathname): + os.mkdir(self.sgf_dir_pathname) + except EnvironmentError: + raise RingmasterError("failed to create SGF directory:\n%s" % e) + + if self.write_gtp_logs: + try: + if not os.path.exists(self.gtplog_dir_pathname): + os.mkdir(self.gtplog_dir_pathname) + except EnvironmentError: + raise RingmasterError( + "failed to create GTP log directory:\n%s" % e) + + def _close_files(self): + """Close the log files.""" + try: + self.logfile.close() + except EnvironmentError, e: + raise RingmasterError("error closing log file:\n%s" % e) + try: + self.historyfile.close() + except EnvironmentError, e: + raise RingmasterError("error closing history file:\n%s" % e) + + ringmaster_settings = [ + Setting('record_games', interpret_bool, True), + Setting('stderr_to_log', interpret_bool, True), + ] + + def _initialise_from_control_file(self, config): + """Interpret the parts of the control file which belong to Ringmaster. + + Sets attributes from ringmaster_settings. + + """ + try: + to_set = load_settings(self.ringmaster_settings, config) + except ValueError, e: + raise ControlFileError(str(e)) + for name, value in to_set.items(): + setattr(self, name, value) + + def enable_gtp_logging(self, b=True): + self.write_gtp_logs = b + + def set_parallel_worker_count(self, n): + self.worker_count = n + + def log(self, s): + print >>self.logfile, s + self.logfile.flush() + + def warn(self, s): + """Log a message and say it on the 'warnings' channel.""" + self.log(s) + self.presenter.say('warnings', s) + + def say(self, channel, s): + """Say a message on the specified channel.""" + self.presenter.say(channel, s) + + def log_history(self, s): + print >>self.historyfile, s + self.historyfile.flush() + + _presenter_classes = { + 'clearing' : ringmaster_presenters.Clearing_presenter, + 'quiet' : ringmaster_presenters.Quiet_presenter, + } + + def set_display_mode(self, presenter_code): + """Specify the presenter to use during run().""" + if presenter_code not in self._presenter_classes: + raise RingmasterError("unknown presenter type: %s" % presenter_code) + self.display_mode = presenter_code + + def _initialise_presenter(self): + self.presenter = self._presenter_classes[self.display_mode]() + + def _initialise_terminal_reader(self): + self.terminal_reader = terminal_input.Terminal_reader() + self.terminal_reader.initialise() + + def get_sgf_filename(self, game_id): + """Return the sgf filename given a game id.""" + return "%s.sgf" % game_id + + def get_sgf_pathname(self, game_id): + """Return the sgf pathname given a game id.""" + return os.path.join(self.sgf_dir_pathname, + self.get_sgf_filename(game_id)) + + + # State attributes (*: in persistent state): + # * void_game_count -- int + # * comp -- from Competition.get_status() + # games_in_progress -- dict game_id -> Game_job + # games_to_replay -- dict game_id -> Game_job + + def _write_status(self, value): + """Write the pickled contents of the persistent state file.""" + f = open(self.status_pathname + ".new", "wb") + pickle.dump(value, f, protocol=-1) + f.close() + os.rename(self.status_pathname + ".new", self.status_pathname) + + def write_status(self): + """Write the persistent state file.""" + competition_status = self.competition.get_status() + status = { + 'void_game_count' : self.void_game_count, + 'comp_vn' : self.competition.status_format_version, + 'comp' : competition_status, + } + try: + self._write_status((self.status_format_version, status)) + except EnvironmentError, e: + raise RingmasterError("error writing persistent state:\n%s" % e) + + def _load_status(self): + """Return the unpickled contents of the persistent state file.""" + with open(self.status_pathname, "rb") as f: + return pickle.load(f) + + def load_status(self): + """Read the persistent state file and load the state it contains.""" + try: + status_format_version, status = self._load_status() + if (status_format_version != self.status_format_version or + status['comp_vn'] != self.competition.status_format_version): + raise StandardError + self.void_game_count = status['void_game_count'] + self.games_in_progress = {} + self.games_to_replay = {} + competition_status = status['comp'] + except pickle.UnpicklingError: + raise RingmasterError("corrupt status file") + except EnvironmentError, e: + raise RingmasterError("error loading status file:\n%s" % e) + except KeyError, e: + raise RingmasterError("incompatible status file: missing %s" % e) + except Exception, e: + # Probably an exception from __setstate__ somewhere + raise RingmasterError("incompatible status file") + try: + self.competition.set_status(competition_status) + except CompetitionError, e: + raise RingmasterError("error loading competition state: %s" % e) + except KeyError, e: + raise RingmasterError( + "error loading competition state: missing %s" % e) + except Exception, e: + raise RingmasterError("error loading competition state:\n%s" % + compact_tracebacks.format_traceback(skip=1)) + self.status_is_loaded = True + + def set_clean_status(self): + """Reset persistent state to the initial values.""" + self.void_game_count = 0 + self.games_in_progress = {} + self.games_to_replay = {} + try: + self.competition.set_clean_status() + except CompetitionError, e: + raise RingmasterError(e) + self.status_is_loaded = True + + def status_file_exists(self): + """Check whether the persistent state file exists.""" + return os.path.exists(self.status_pathname) + + def print_status(self): + """Print the contents of the persistent state file, for debugging.""" + from pprint import pprint + status_format_version, status = self._load_status() + print "status_format_version:", status_format_version + pprint(status) + + def write_command(self, command): + """Write a command to the command file. + + command -- short string + + Overwrites the command file if it already exists. + + """ + # Short enough that I think we can get aw + try: + f = open(self.command_pathname, "w") + f.write(command) + f.close() + except EnvironmentError, e: + raise RingmasterError("error writing command file:\n%s" % e) + + def get_tournament_results(self): + """Provide access to the tournament's results. + + Returns a Tournament_results object. + + Raises RingmasterError if the competition state isn't loaded, or if the + competition isn't a tournament. + + """ + if not self.status_is_loaded: + raise RingmasterError("status is not loaded") + try: + return self.competition.get_tournament_results() + except NotImplementedError: + raise RingmasterError("competition is not a tournament") + + def report(self): + """Write the full competition report to the report file.""" + f = open(self.report_pathname, "w") + self.competition.write_full_report(f) + f.close() + + def print_status_report(self): + """Write current competition status to standard output. + + This is for the 'show' command. + + """ + self.competition.write_short_report(sys.stdout) + + def _halt_competition(self, reason): + """Make the competition stop submitting new games. + + reason -- message for the log and the status box. + + """ + self.stopping = True + self.stopping_reason = reason + self.log("halting competition: %s" % reason) + + def _update_display(self): + """Redisplay the 'live' competition description. + + Does nothing in quiet mode. + + """ + if self.presenter.shows_warnings_only: + return + def p(s): + self.say('status', s) + self.presenter.clear('status') + if self.stopping: + if self.worker_count is None or not self.games_in_progress: + p("halting: %s" % self.stopping_reason) + else: + p("waiting for workers to finish: %s" % + self.stopping_reason) + if self.games_in_progress: + if self.worker_count is None: + gms = "game" + else: + gms = "%d games" % len(self.games_in_progress) + p("%s in progress: %s" % + (gms, " ".join(sorted(self.games_in_progress)))) + if not self.stopping: + if self.max_games_this_run is not None: + p("will start at most %d more games in this run" % + self.max_games_this_run) + if self.terminal_reader.is_enabled(): + p("(Ctrl-X to halt gracefully)") + + self.presenter.clear('screen_report') + sr = self.presenter.get_stream('screen_report') + if self.void_game_count > 0: + print >>sr, "%d void games; see log file." % self.void_game_count + self.competition.write_screen_report(sr) + sr.close() + + self.presenter.refresh() + + def _prepare_job(self, job): + """Finish off a Game_job provided by the Competition. + + job -- incomplete Game_job, as returned by Competition.get_game() + + """ + job.sgf_game_name = "%s %s" % (self.competition_code, job.game_id) + if self.record_games: + job.sgf_filename = self.get_sgf_filename(job.game_id) + job.sgf_dirname = self.sgf_dir_pathname + job.void_sgf_dirname = self.void_dir_pathname + if self.write_gtp_logs: + job.gtp_log_pathname = os.path.join( + self.gtplog_dir_pathname, "%s.log" % job.game_id) + if self.stderr_to_log: + job.stderr_pathname = self.log_pathname + + def get_job(self): + """Job supply function for the job manager.""" + job = self._get_job() + self._update_display() + return job + + def _get_job(self): + """Main implementation of get_job().""" + + if self.stopping: + return job_manager.NoJobAvailable + + if self.terminal_reader.stop_was_requested(): + self._halt_competition("stop instruction received from terminal") + if self.presenter.shows_warnings_only: + self.terminal_reader.acknowledge() + return job_manager.NoJobAvailable + + try: + if os.path.exists(self.command_pathname): + command = open(self.command_pathname).read() + if command == "stop": + self._halt_competition("stop command received") + try: + os.remove(self.command_pathname) + except EnvironmentError, e: + self.warn("error removing .cmd file:\n%s" % e) + return job_manager.NoJobAvailable + except EnvironmentError, e: + self.warn("error reading .cmd file:\n%s" % e) + if self.max_games_this_run is not None: + if self.max_games_this_run == 0: + self._halt_competition("max-games reached for this run") + return job_manager.NoJobAvailable + self.max_games_this_run -= 1 + + if self.games_to_replay: + _, job = self.games_to_replay.popitem() + else: + job = self.competition.get_game() + if job is NoGameAvailable: + return job_manager.NoJobAvailable + if job.game_id in self.games_in_progress: + raise RingmasterInternalError( + "duplicate game id: %s" % job.game_id) + self._prepare_job(job) + self.games_in_progress[job.game_id] = job + start_msg = "starting game %s: %s (b) vs %s (w)" % ( + job.game_id, job.player_b.code, job.player_w.code) + self.log(start_msg) + + return job + + def process_response(self, response): + """Job response function for the job manager.""" + # We log before processing the result, in case there's an error from the + # competition code. + self.log("response from game %s" % response.game_id) + for warning in response.warnings: + self.warn(warning) + for log_entry in response.log_entries: + self.log(log_entry) + result_description = self.competition.process_game_result(response) + del self.games_in_progress[response.game_id] + self.write_status() + if result_description is None: + result_description = response.game_result.describe() + self.say('results', "game %s: %s" % ( + response.game_id, result_description)) + + def process_error_response(self, job, message): + """Job error response function for the job manager.""" + self.warn("game %s -- %s" % ( + job.game_id, message)) + self.void_game_count += 1 + previous_error_count = self.game_error_counts.get(job.game_id, 0) + stop_competition, retry_game = \ + self.competition.process_game_error(job, previous_error_count) + if retry_game and not stop_competition: + self.games_to_replay[job.game_id] = \ + self.games_in_progress.pop(job.game_id) + self.game_error_counts[job.game_id] = previous_error_count + 1 + else: + del self.games_in_progress[job.game_id] + if previous_error_count != 0: + del self.game_error_counts[job.game_id] + self.write_status() + if stop_competition and not self.stopping: + # No need to log: _halt competition will do so + self.say('warnings', "halting run due to void games") + self._halt_competition("too many void games") + + def run(self, max_games=None): + """Run the competition. + + max_games -- int or None (maximum games to start in this run) + + Returns when max_games have been played in this run, when the + Competition is over, or when a 'stop' command is received via the + command file. + + """ + def now(): + return datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + + def log_games_in_progress(): + try: + msg = "games in progress were: %s" % ( + " ".join(sorted(self.games_in_progress))) + except Exception: + pass + self.log(msg) + + self._open_files() + self.competition.set_event_logger(self.log) + self.competition.set_history_logger(self.log_history) + + self._initialise_presenter() + self._initialise_terminal_reader() + + allow_mp = (self.worker_count is not None) + self.log("run started at %s with max_games %s" % (now(), max_games)) + if allow_mp: + self.log("using %d worker processes" % self.worker_count) + self.max_games_this_run = max_games + self._update_display() + try: + job_manager.run_jobs( + job_source=self, + allow_mp=allow_mp, max_workers=self.worker_count, + passed_exceptions=[RingmasterError, CompetitionError, + RingmasterInternalError]) + except KeyboardInterrupt: + self.log("run interrupted at %s" % now()) + log_games_in_progress() + raise + except (RingmasterError, CompetitionError), e: + self.log("run finished with error at %s\n%s" % (now(), e)) + log_games_in_progress() + raise RingmasterError(e) + except (job_manager.JobSourceError, RingmasterInternalError), e: + self.log("run finished with internal error at %s\n%s" % (now(), e)) + log_games_in_progress() + raise RingmasterInternalError(e) + except: + self.log("run finished with internal error at %s" % now()) + self.log(compact_tracebacks.format_traceback()) + log_games_in_progress() + raise + self.log("run finished at %s" % now()) + self._close_files() + + def delete_state_and_output(self): + """Delete all files generated by this competition. + + Deletes the persistent state file, game records, log files, and reports. + + """ + for pathname in [ + self.log_pathname, + self.status_pathname, + self.command_pathname, + self.history_pathname, + self.report_pathname, + ]: + if os.path.exists(pathname): + try: + os.remove(pathname) + except EnvironmentError, e: + print >>sys.stderr, e + for pathname in [ + self.sgf_dir_pathname, + self.void_dir_pathname, + self.gtplog_dir_pathname, + ]: + if os.path.exists(pathname): + try: + shutil.rmtree(pathname) + except EnvironmentError, e: + print >>sys.stderr, e + + def check_players(self, discard_stderr=False): + """Check that the engines required for the competition will run. + + If an engine fails, prints a description of the problem and returns + False without continuing to check. + + Otherwise returns True. + + """ + try: + to_check = self.competition.get_player_checks() + except CompetitionError, e: + raise RingmasterError(e) + for check in to_check: + if not discard_stderr: + print "checking player %s" % check.player.code + try: + msgs = game_jobs.check_player(check, discard_stderr) + except game_jobs.CheckFailed, e: + print "player %s failed startup check:\n%s" % ( + check.player.code, e) + return False + else: + if not discard_stderr: + for msg in msgs: + print msg + return True + diff --git a/gomill/gomill/settings.py b/gomill/gomill/settings.py new file mode 100644 index 0000000..9595be0 --- /dev/null +++ b/gomill/gomill/settings.py @@ -0,0 +1,436 @@ +"""Support for describing configurable values.""" + +import re +import shlex + +__all__ = ['Setting', 'allow_none', 'load_settings', + 'Config_proxy', 'Quiet_config', + 'interpret_any', 'interpret_bool', + 'interpret_int', 'interpret_positive_int', 'interpret_float', + 'interpret_8bit_string', 'interpret_identifier', + 'interpret_as_utf8', 'interpret_as_utf8_stripped', + 'interpret_colour', 'interpret_enum', 'interpret_callable', + 'interpret_shlex_sequence', + 'interpret_sequence', 'interpret_sequence_of', + 'interpret_sequence_of_quiet_configs', + 'interpret_map', 'interpret_map_of', + 'clean_string', + ] + +def interpret_any(v): + return v + +def interpret_bool(b): + if b is not True and b is not False: + raise ValueError("invalid True/False value") + return b + +def interpret_int(i): + if not isinstance(i, int) or isinstance(i, long): + raise ValueError("invalid integer") + return i + +def interpret_positive_int(i): + if not isinstance(i, int) or isinstance(i, long): + raise ValueError("invalid integer") + if i <= 0: + raise ValueError("must be positive integer") + return i + +def interpret_float(f): + if isinstance(f, float): + return f + if isinstance(f, int) or isinstance(f, long): + return float(f) + raise ValueError("invalid float") + +def interpret_8bit_string(s): + if isinstance(s, str): + result = s + elif isinstance(s, unicode): + try: + result = s.encode("ascii") + except UnicodeEncodeError: + raise ValueError("non-ascii character in unicode string") + else: + raise ValueError("not a string") + if '\0' in s: + raise ValueError("contains NUL") + return result + +def interpret_as_utf8(s): + if isinstance(s, str): + try: + s.decode("utf-8") + except UnicodeDecodeError: + raise ValueError("not a valid utf-8 string") + return s + if isinstance(s, unicode): + return s.encode("utf-8") + if s is None: + return "" + raise ValueError("invalid string") + +def interpret_as_utf8_stripped(s): + return interpret_as_utf8(s).strip() + + +def clean_string(s): + return re.sub(r"[\x00-\x1f\x7f-\x9f]", "?", s) + +# NB, tuners use '#' in player codes +_identifier_re = re.compile(r"\A[-!$%&*+-.:;<=>?^_~a-zA-Z0-9]*\Z") + +def interpret_identifier(s): + if isinstance(s, unicode): + try: + s = s.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "contains forbidden character: %s" % + clean_string(s.encode("ascii", "replace"))) + elif not isinstance(s, str): + raise ValueError("not a string") + if not s: + raise ValueError("empty string") + if not _identifier_re.search(s): + raise ValueError("contains forbidden character: %s" % clean_string(s)) + return s + +_colour_dict = { + 'b' : 'b', + 'black' : 'b', + 'w' : 'w', + 'white' : 'w', + } + +def interpret_colour(s): + if isinstance(s, basestring): + try: + return _colour_dict[s.lower()] + except KeyError: + pass + raise ValueError("invalid colour") + +def interpret_enum(*values): + def interpreter(value): + if value not in values: + raise ValueError("unknown value") + return value + return interpreter + +def interpret_callable(c): + if not callable(c): + raise ValueError("invalid callable") + return c + +def interpret_shlex_sequence(v): + """Interpret a sequence of 'shlex' tokens. + + If v is a string, calls shlex.split() on it. + + Otherwise, treats it as a list of strings. + + Rejects empty sequences. + + """ + if isinstance(v, basestring): + result = shlex.split(interpret_8bit_string(v)) + else: + try: + l = interpret_sequence(v) + except ValueError: + raise ValueError("not a string or a sequence") + try: + result = [interpret_8bit_string(s) for s in l] + except ValueError, e: + raise ValueError("element %s" % e) + if not result: + raise ValueError("empty") + return result + + +def interpret_sequence(l): + """Interpret a list-like object. + + Accepts any iterable and returns a list. + + """ + try: + l = list(l) + except Exception: + raise ValueError("not a sequence") + return l + +def interpret_sequence_of(item_interpreter): + """Make an interpreter for list-like objects. + + The interpreter behaves like interpret_list, and additionally calls + item_interpreter for each list item. + + """ + def interpreter(value): + l = interpret_sequence(value) + for i, v in enumerate(l): + try: + l[i] = item_interpreter(v) + except ValueError, e: + raise ValueError("item %s: %s" % (i, e)) + return l + return interpreter + +def interpret_sequence_of_quiet_configs(cls, allow_simple_values=False): + """Make an interpreter for sequences of a given Quiet_config. + + If 'allow_simple_values' is true, any value which isn't an instance of 'cls' + will be used (as a single positional parameter) to instantiate a 'cls' + instance. + + """ + def interpret(v): + if not isinstance(v, cls): + if allow_simple_values: + v = cls(v) + else: + raise ValueError("not a %s" % cls.get_type_name()) + return v + return interpret_sequence_of(interpret) + +def interpret_map(m): + """Interpret a map-like object. + + Accepts anything that dict() accepts for its first argument. + + Returns a list of pairs (key, value). + + """ + try: + d = dict(m) + except Exception: + raise ValueError("not a map") + return d.items() + +def interpret_map_of(key_interpreter, value_interpreter): + """Make an interpreter for map-like objects. + + The interpreter behaves like interpret_map, and additionally calls + key_interpreter for each key and value_interpreter for each value. + + Sorts the result by key. + + """ + def interpreter(m): + result = [] + for key, value in interpret_map(m): + try: + new_key = key_interpreter(key) + except ValueError, e: + raise ValueError("bad key: %s" % e) + try: + new_value = value_interpreter(value) + except ValueError, e: + # we assume validated keys are fit to print + raise ValueError("bad value for '%s': %s" % (new_key, e)) + result.append((new_key, new_value)) + # We assume validated items are suitable for sorting + return sorted(result) + return interpreter + +def allow_none(fn): + """Make a new interpreter from an existing one, which maps None to None.""" + def sub(v): + if v is None: + return None + return fn(v) + return sub + +_nodefault = object() + +class Setting(object): + """Describe a single setting. + + Instantiate with: + setting name + interpreter function + optionally: + default value, or + defaultmaker -- callable creating the default value + + """ + def __init__(self, name, interpreter, + default=_nodefault, defaultmaker=None): + self.name = name + self.interpreter = interpreter + self.default = default + self.defaultmaker = defaultmaker + + def get_default(self): + """Return the default value for this setting. + + Raises KeyError if there isn't one. + + """ + if self.default is not _nodefault: + return self.default + if self.defaultmaker is not None: + return self.defaultmaker() + raise KeyError + + def interpret(self, value): + """Validate the value and normalise if necessary. + + Returns the normalised value (usually unchanged). + + Raises ValueError with a description if the value is invalid. + + """ + try: + return self.interpreter(value) + except ValueError, e: + raise ValueError("'%s': %s" % (self.name, e)) + +def load_settings(settings, config, apply_defaults=True, allow_missing=False): + """Read settings values from configuration. + + settings -- list of Settings + config -- dict containing the values to be read + apply_defaults -- bool (default true) + allow_missing -- bool (default false) + + Returns a dict: setting name -> interpreted value + + Handling of values which aren't present in 'config': + - if apply_defaults is true, the setting's default is substituted + - if apply_defaults is false or the setting has no default: + - if allow_missing is true, omits the setting from the returned dict + - if allow_missing is false, raises ValueError + + Resolves Config_proxy objects (see below) + + Raises ValueError with a description if a value can't be interpreted. + + """ + result = {} + for setting in settings: + try: + try: + v = config[setting.name] + if isinstance(v, Config_proxy): + try: + v = v.resolve() + except ValueError, e: + raise ValueError("'%s': %s" % (setting.name, e)) + # May propagate ValueError + v = setting.interpret(v) + except KeyError: + if apply_defaults: + v = setting.get_default() + else: + raise + except KeyError: + if allow_missing: + continue + else: + raise ValueError("'%s' not specified" % setting.name) + result[setting.name] = v + return result + +class Config_proxy(object): + """Class proxy for use in control files. + + To use this, define a subclass, giving it the following class attribute: + underlying -- the underlying class + + Then in the control file, the proxy can be used anywhere which will be + interpreted using the settings mechanism. An instance of the underlying + class will be created by load_settings and then passed to the interpret + function as usual. + + Any errors from the underlying class's __init__ will be raised as ValueError + from load_settings(). + + """ + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def resolve(self): + try: + return self.underlying(*self.args, **self.kwargs) + except Exception, e: + raise ValueError("invalid parameters for %s:\n%s" % + (self.__class__.__name__, e)) + + +class Quiet_config(object): + """Configuration object for use in control files. + + At instantiation time, this just records its arguments, so they can be + validated later. + + """ + # These may be specified as positional or keyword + positional_arguments = () + # These are keyword-only + keyword_arguments = () + # Used by interpret_sequence_of_quiet_configs + type_name = None + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + @classmethod + def get_type_name(cls): + """Return a name for the config type, for use in error messages.""" + if cls.type_name is not None: + return cls.type_name + return cls.__name__.partition("_config")[0] + + def resolve_arguments(self): + """Combine positional and keyword arguments. + + Returns a dict: argument name -> value + + Raises ValueError if the arguments are invalid. + + Checks for: + - too many positional arguments + - unknown keyword arguments + - argument specified as both positional and keyword + + Unspecified arguments (either positional or keyword) are not considered + errors; they're just not included in the result. + + """ + result = {} + if len(self.args) > len(self.positional_arguments): + raise ValueError("too many positional arguments") + for name, val in zip(self.positional_arguments, self.args): + result[name] = val + allowed = set(self.positional_arguments + self.keyword_arguments) + for name, val in sorted(self.kwargs.iteritems()): + if name not in allowed: + raise ValueError("unknown argument '%s'" % name) + if name in result: + raise ValueError( + "%s specified as both positional and keyword argument" % + name) + result[name] = val + return result + + def get_key(self): + """Retrieve the first positional argument, if possible. + + Does the right thing if it was specified as a keyword argument. + + Returns None if there isn't one. + + """ + try: + if self.args: + return self.args[0] + return self.kwargs[self.positional_arguments[0]] + except LookupError: + return None + diff --git a/gomill/gomill/sgf.py b/gomill/gomill/sgf.py new file mode 100644 index 0000000..7cb8b3e --- /dev/null +++ b/gomill/gomill/sgf.py @@ -0,0 +1,806 @@ +"""Represent SGF games. + +This is intended for use with SGF FF[4]; see http://www.red-bean.com/sgf/ + +""" + +import datetime + +from gomill import sgf_grammar +from gomill import sgf_properties + + +class Node(object): + """An SGF node. + + Instantiate with a raw property map (see sgf_grammar) and an + sgf_properties.Presenter. + + A Node doesn't belong to a particular game (cf Tree_node below), but it + knows its board size (in order to interpret move values) and the encoding + to use for the raw property strings. + + Changing the SZ property isn't allowed. + + """ + def __init__(self, property_map, presenter): + # Map identifier (PropIdent) -> nonempty list of raw values + self._property_map = property_map + self._presenter = presenter + + def get_size(self): + """Return the board size used to interpret property values.""" + return self._presenter.size + + def get_encoding(self): + """Return the encoding used for raw property values. + + Returns a string (a valid Python codec name, eg "UTF-8"). + + """ + return self._presenter.encoding + + def get_presenter(self): + """Return the node's sgf_properties.Presenter.""" + return self._presenter + + def has_property(self, identifier): + """Check whether the node has the specified property.""" + return identifier in self._property_map + + def properties(self): + """Find the properties defined for the node. + + Returns a list of property identifiers, in unspecified order. + + """ + return self._property_map.keys() + + def get_raw_list(self, identifier): + """Return the raw values of the specified property. + + Returns a nonempty list of 8-bit strings, in the raw property encoding. + + The strings contain the exact bytes that go between the square brackets + (without interpreting escapes or performing any whitespace conversion). + + Raises KeyError if there was no property with the given identifier. + + (If the property is an empty elist, this returns a list containing a + single empty string.) + + """ + return self._property_map[identifier] + + def get_raw(self, identifier): + """Return a single raw value of the specified property. + + Returns an 8-bit string, in the raw property encoding. + + The string contains the exact bytes that go between the square brackets + (without interpreting escapes or performing any whitespace conversion). + + Raises KeyError if there was no property with the given identifier. + + If the property has multiple values, this returns the first (if the + value is an empty elist, this returns an empty string). + + """ + return self._property_map[identifier][0] + + def get_raw_property_map(self): + """Return the raw values of all properties as a dict. + + Returns a dict mapping property identifiers to lists of raw values + (see get_raw_list()). + + Returns the same dict each time it's called. + + Treat the returned dict as read-only. + + """ + return self._property_map + + + def _set_raw_list(self, identifier, values): + if identifier == "SZ" and values != [str(self._presenter.size)]: + raise ValueError("changing size is not permitted") + self._property_map[identifier] = values + + def unset(self, identifier): + """Remove the specified property. + + Raises KeyError if the property isn't currently present. + + """ + if identifier == "SZ" and self._presenter.size != 19: + raise ValueError("changing size is not permitted") + del self._property_map[identifier] + + + def set_raw_list(self, identifier, values): + """Set the raw values of the specified property. + + identifier -- ascii string passing is_valid_property_identifier() + values -- nonempty iterable of 8-bit strings in the raw property + encoding + + The values specify the exact bytes to appear between the square + brackets in the SGF file; you must perform any necessary escaping + first. + + (To specify an empty elist, pass a list containing a single empty + string.) + + """ + if not sgf_grammar.is_valid_property_identifier(identifier): + raise ValueError("ill-formed property identifier") + values = list(values) + if not values: + raise ValueError("empty property list") + for value in values: + if not sgf_grammar.is_valid_property_value(value): + raise ValueError("ill-formed raw property value") + self._set_raw_list(identifier, values) + + def set_raw(self, identifier, value): + """Set the specified property to a single raw value. + + identifier -- ascii string passing is_valid_property_identifier() + value -- 8-bit string in the raw property encoding + + The value specifies the exact bytes to appear between the square + brackets in the SGF file; you must perform any necessary escaping + first. + + """ + if not sgf_grammar.is_valid_property_identifier(identifier): + raise ValueError("ill-formed property identifier") + if not sgf_grammar.is_valid_property_value(value): + raise ValueError("ill-formed raw property value") + self._set_raw_list(identifier, [value]) + + + def get(self, identifier): + """Return the interpreted value of the specified property. + + Returns the value as a suitable Python representation. + + Raises KeyError if the node does not have a property with the given + identifier. + + Raises ValueError if it cannot interpret the value. + + See sgf_properties.Presenter.interpret() for details. + + """ + return self._presenter.interpret( + identifier, self._property_map[identifier]) + + def set(self, identifier, value): + """Set the value of the specified property. + + identifier -- ascii string passing is_valid_property_identifier() + value -- new property value (in its Python representation) + + For properties with value type 'none', use value True. + + Raises ValueError if it cannot represent the value. + + See sgf_properties.Presenter.serialise() for details. + + """ + self._set_raw_list( + identifier, self._presenter.serialise(identifier, value)) + + def get_raw_move(self): + """Return the raw value of the move from a node. + + Returns a pair (colour, raw value) + + colour is 'b' or 'w'. + + Returns None, None if the node contains no B or W property. + + """ + values = self._property_map.get("B") + if values is not None: + colour = "b" + else: + values = self._property_map.get("W") + if values is not None: + colour = "w" + else: + return None, None + return colour, values[0] + + def get_move(self): + """Retrieve the move from a node. + + Returns a pair (colour, move) + + colour is 'b' or 'w'. + + move is (row, col), or None for a pass. + + Returns None, None if the node contains no B or W property. + + """ + colour, raw = self.get_raw_move() + if colour is None: + return None, None + return (colour, + sgf_properties.interpret_go_point(raw, self._presenter.size)) + + def get_setup_stones(self): + """Retrieve Add Black / Add White / Add Empty properties from a node. + + Returns a tuple (black_points, white_points, empty_points) + + Each value is a set of pairs (row, col). + + """ + try: + bp = self.get("AB") + except KeyError: + bp = set() + try: + wp = self.get("AW") + except KeyError: + wp = set() + try: + ep = self.get("AE") + except KeyError: + ep = set() + return bp, wp, ep + + def has_setup_stones(self): + """Check whether the node has any AB/AW/AE properties.""" + d = self._property_map + return ("AB" in d or "AW" in d or "AE" in d) + + def set_move(self, colour, move): + """Set the B or W property. + + colour -- 'b' or 'w'. + move -- (row, col), or None for a pass. + + Replaces any existing B or W property in the node. + + """ + if colour not in ('b', 'w'): + raise ValueError + if 'B' in self._property_map: + del self._property_map['B'] + if 'W' in self._property_map: + del self._property_map['W'] + self.set(colour.upper(), move) + + def set_setup_stones(self, black, white, empty=None): + """Set Add Black / Add White / Add Empty properties. + + black, white, empty -- list or set of pairs (row, col) + + Removes any existing AB/AW/AE properties from the node. + + """ + if 'AB' in self._property_map: + del self._property_map['AB'] + if 'AW' in self._property_map: + del self._property_map['AW'] + if 'AE' in self._property_map: + del self._property_map['AE'] + if black: + self.set('AB', black) + if white: + self.set('AW', white) + if empty: + self.set('AE', empty) + + def add_comment_text(self, text): + """Add or extend the node's comment. + + If the node doesn't have a C property, adds one with the specified + text. + + Otherwise, adds the specified text to the existing C property value + (with two newlines in front). + + """ + if self.has_property('C'): + self.set('C', self.get('C') + "\n\n" + text) + else: + self.set('C', text) + + def __str__(self): + def format_property(ident, values): + return ident + "".join("[%s]" % s for s in values) + return "\n".join( + format_property(ident, values) + for (ident, values) in sorted(self._property_map.items())) \ + + "\n" + + +class Tree_node(Node): + """A node embedded in an SGF game. + + A Tree_node is a Node that also knows its position within an Sgf_game. + + Do not instantiate directly; retrieve from an Sgf_game or another Tree_node. + + A Tree_node is a list-like container of its children: it can be indexed, + sliced, and iterated over like a list, and supports index(). + + A Tree_node with no children is treated as having truth value false. + + Public attributes (treat as read-only): + owner -- the node's Sgf_game + parent -- the nodes's parent Tree_node (None for the root node) + + """ + def __init__(self, parent, properties): + self.owner = parent.owner + self.parent = parent + self._children = [] + Node.__init__(self, properties, parent._presenter) + + def _add_child(self, node): + self._children.append(node) + + def __len__(self): + return len(self._children) + + def __getitem__(self, key): + return self._children[key] + + def index(self, child): + return self._children.index(child) + + def new_child(self, index=None): + """Create a new Tree_node and add it as this node's last child. + + If 'index' is specified, the new node is inserted in the child list at + the specified index instead (behaves like list.insert). + + Returns the new node. + + """ + child = Tree_node(self, {}) + if index is None: + self._children.append(child) + else: + self._children.insert(index, child) + return child + + def delete(self): + """Remove this node from its parent.""" + if self.parent is None: + raise ValueError("can't remove the root node") + self.parent._children.remove(self) + + def reparent(self, new_parent, index=None): + """Move this node to a new place in the tree. + + new_parent -- Tree_node from the same game. + + Raises ValueError if the new parent is this node or one of its + descendants. + + If 'index' is specified, the node is inserted in the new parent's child + list at the specified index (behaves like list.insert); otherwise it's + placed at the end. + + """ + if new_parent.owner != self.owner: + raise ValueError("new parent doesn't belong to the same game") + n = new_parent + while True: + if n == self: + raise ValueError("would create a loop") + n = n.parent + if n is None: + break + # self.parent is not None because moving the root would create a loop. + self.parent._children.remove(self) + self.parent = new_parent + if index is None: + new_parent._children.append(self) + else: + new_parent._children.insert(index, self) + + def find(self, identifier): + """Find the nearest ancestor-or-self containing the specified property. + + Returns a Tree_node, or None if there is no such node. + + """ + node = self + while node is not None: + if node.has_property(identifier): + return node + node = node.parent + return None + + def find_property(self, identifier): + """Return the value of a property, defined at this node or an ancestor. + + This is intended for use with properties of type 'game-info', and with + properties with the 'inherit' attribute. + + This returns the interpreted value, in the same way as get(). + + It searches up the tree, in the same way as find(). + + Raises KeyError if no node defining the property is found. + + """ + node = self.find(identifier) + if node is None: + raise KeyError + return node.get(identifier) + +class _Root_tree_node(Tree_node): + """Variant of Tree_node used for a game root.""" + def __init__(self, property_map, owner): + self.owner = owner + self.parent = None + self._children = [] + Node.__init__(self, property_map, owner.presenter) + +class _Unexpanded_root_tree_node(_Root_tree_node): + """Variant of _Root_tree_node used with 'loaded' Sgf_games.""" + def __init__(self, owner, coarse_tree): + _Root_tree_node.__init__(self, coarse_tree.sequence[0], owner) + self._coarse_tree = coarse_tree + + def _expand(self): + sgf_grammar.make_tree( + self._coarse_tree, self, Tree_node, Tree_node._add_child) + delattr(self, '_coarse_tree') + self.__class__ = _Root_tree_node + + def __len__(self): + self._expand() + return self.__len__() + + def __getitem__(self, key): + self._expand() + return self.__getitem__(key) + + def index(self, child): + self._expand() + return self.index(child) + + def new_child(self): + self._expand() + return self.new_child() + + def _main_sequence_iter(self): + presenter = self._presenter + for properties in sgf_grammar.main_sequence_iter(self._coarse_tree): + yield Node(properties, presenter) + + +class Sgf_game(object): + """An SGF game tree. + + The complete game tree is represented using Tree_nodes. The various methods + which return Tree_nodes will always return the same object for the same + node. + + Instantiate with + size -- int (board size), in range 1 to 26 + encoding -- the raw property encoding (default "UTF-8") + + 'encoding' must be a valid Python codec name. + + The following root node properties are set for newly-created games: + FF[4] + GM[1] + SZ[size] + CA[encoding] + + Changing FF and GM is permitted (but this library will carry on using the + FF[4] and GM[1] rules). Changing SZ is not permitted (unless the change + leaves the effective value unchanged). Changing CA is permitted; this + controls the encoding used by serialise(). + + """ + def __new__(cls, size, encoding="UTF-8", *args, **kwargs): + # To complete initialisation after this, you need to set 'root'. + if not 1 <= size <= 26: + raise ValueError("size out of range: %s" % size) + game = super(Sgf_game, cls).__new__(cls) + game.size = size + game.presenter = sgf_properties.Presenter(size, encoding) + return game + + def __init__(self, *args, **kwargs): + self.root = _Root_tree_node({}, self) + self.root.set_raw('FF', "4") + self.root.set_raw('GM', "1") + self.root.set_raw('SZ', str(self.size)) + # Read the encoding back so we get the normalised form + self.root.set_raw('CA', self.presenter.encoding) + + @classmethod + def from_coarse_game_tree(cls, coarse_game, override_encoding=None): + """Alternative constructor: create an Sgf_game from the parser output. + + coarse_game -- Coarse_game_tree + override_encoding -- encoding name, eg "UTF-8" (optional) + + The nodes' property maps (as returned by get_raw_property_map()) will + be the same dictionary objects as the ones from the Coarse_game_tree. + + The board size and raw property encoding are taken from the SZ and CA + properties in the root node (defaulting to 19 and "ISO-8859-1", + respectively). + + If override_encoding is specified, the source data is assumed to be in + the specified encoding (no matter what the CA property says), and the + CA property is set to match. + + """ + try: + size_s = coarse_game.sequence[0]['SZ'][0] + except KeyError: + size = 19 + else: + try: + size = int(size_s) + except ValueError: + raise ValueError("bad SZ property: %s" % size_s) + if override_encoding is None: + try: + encoding = coarse_game.sequence[0]['CA'][0] + except KeyError: + encoding = "ISO-8859-1" + else: + encoding = override_encoding + game = cls.__new__(cls, size, encoding) + game.root = _Unexpanded_root_tree_node(game, coarse_game) + if override_encoding is not None: + game.root.set_raw("CA", game.presenter.encoding) + return game + + @classmethod + def from_string(cls, s, override_encoding=None): + """Alternative constructor: read a single Sgf_game from a string. + + s -- 8-bit string + + Raises ValueError if it can't parse the string. See parse_sgf_game() + for details. + + See from_coarse_game_tree for details of size and encoding handling. + + """ + coarse_game = sgf_grammar.parse_sgf_game(s) + return cls.from_coarse_game_tree(coarse_game, override_encoding) + + def serialise(self, wrap=79): + """Serialise the SGF data as a string. + + wrap -- int (default 79), or None + + Returns an 8-bit string, in the encoding specified by the CA property + in the root node (defaulting to "ISO-8859-1"). + + If the raw property encoding and the target encoding match (which is + the usual case), the raw property values are included unchanged in the + output (even if they are improperly encoded.) + + Otherwise, if any raw property value is improperly encoded, + UnicodeDecodeError is raised, and if any property value can't be + represented in the target encoding, UnicodeEncodeError is raised. + + If the target encoding doesn't identify a Python codec, ValueError is + raised. Behaviour is unspecified if the target encoding isn't + ASCII-compatible (eg, UTF-16). + + If 'wrap' is not None, makes some effort to keep output lines no longer + than 'wrap'. + + """ + try: + encoding = self.get_charset() + except ValueError: + raise ValueError("unsupported charset: %s" % + self.root.get_raw_list("CA")) + coarse_tree = sgf_grammar.make_coarse_game_tree( + self.root, lambda node:node, Node.get_raw_property_map) + serialised = sgf_grammar.serialise_game_tree(coarse_tree, wrap) + if encoding == self.root.get_encoding(): + return serialised + else: + return serialised.decode(self.root.get_encoding()).encode(encoding) + + + def get_property_presenter(self): + """Return the property presenter. + + Returns an sgf_properties.Presenter. + + This can be used to customise how property values are interpreted and + serialised. + + """ + return self.presenter + + def get_root(self): + """Return the root node (as a Tree_node).""" + return self.root + + def get_last_node(self): + """Return the last node in the 'leftmost' variation (as a Tree_node).""" + node = self.root + while node: + node = node[0] + return node + + def get_main_sequence(self): + """Return the 'leftmost' variation. + + Returns a list of Tree_nodes, from the root to a leaf. + + """ + node = self.root + result = [node] + while node: + node = node[0] + result.append(node) + return result + + def get_main_sequence_below(self, node): + """Return the 'leftmost' variation below the specified node. + + node -- Tree_node + + Returns a list of Tree_nodes, from the first child of 'node' to a leaf. + + """ + if node.owner is not self: + raise ValueError("node doesn't belong to this game") + result = [] + while node: + node = node[0] + result.append(node) + return result + + def get_sequence_above(self, node): + """Return the partial variation leading to the specified node. + + node -- Tree_node + + Returns a list of Tree_nodes, from the root to the parent of 'node'. + + """ + if node.owner is not self: + raise ValueError("node doesn't belong to this game") + result = [] + while node.parent is not None: + node = node.parent + result.append(node) + result.reverse() + return result + + def main_sequence_iter(self): + """Provide the 'leftmost' variation as an iterator. + + Returns an iterator providing Node instances, from the root to a leaf. + + The Node instances may or may not be Tree_nodes. + + It's OK to use these Node instances to modify properties: even if they + are not the same objects as returned by the main tree navigation + methods, they share the underlying property maps. + + If you know the game has no variations, or you're only interested in + the 'leftmost' variation, you can use this function to retrieve the + nodes without building the entire game tree. + + """ + if isinstance(self.root, _Unexpanded_root_tree_node): + return self.root._main_sequence_iter() + return iter(self.get_main_sequence()) + + def extend_main_sequence(self): + """Create a new Tree_node and add to the 'leftmost' variation. + + Returns the new node. + + """ + return self.get_last_node().new_child() + + def get_size(self): + """Return the board size as an integer.""" + return self.size + + def get_charset(self): + """Return the effective value of the CA root property. + + This applies the default, and returns the normalised form. + + Raises ValueError if the CA property doesn't identify a Python codec. + + """ + try: + s = self.root.get("CA") + except KeyError: + return "ISO-8859-1" + try: + return sgf_properties.normalise_charset_name(s) + except LookupError: + raise ValueError("no codec available for CA %s" % s) + + def get_komi(self): + """Return the komi as a float. + + Returns 0.0 if the KM property isn't present in the root node. + + Raises ValueError if the KM property is malformed. + + """ + try: + return self.root.get("KM") + except KeyError: + return 0.0 + + def get_handicap(self): + """Return the number of handicap stones as a small integer. + + Returns None if the HA property isn't present, or has (illegal) value + zero. + + Raises ValueError if the HA property is otherwise malformed. + + """ + try: + handicap = self.root.get("HA") + except KeyError: + return None + if handicap == 0: + handicap = None + elif handicap == 1: + raise ValueError + return handicap + + def get_player_name(self, colour): + """Return the name of the specified player. + + Returns None if there is no corresponding 'PB' or 'PW' property. + + """ + try: + return self.root.get({'b' : 'PB', 'w' : 'PW'}[colour]) + except KeyError: + return None + + def get_winner(self): + """Return the colour of the winning player. + + Returns None if there is no RE property, or if neither player won. + + """ + try: + colour = self.root.get("RE")[0].lower() + except LookupError: + return None + if colour not in ("b", "w"): + return None + return colour + + def set_date(self, date=None): + """Set the DT property to a single date. + + date -- datetime.date (defaults to today) + + (SGF allows dates to be rather more complicated than this, so there's + no corresponding get_date() method.) + + """ + if date is None: + date = datetime.date.today() + self.root.set('DT', date.strftime("%Y-%m-%d")) + diff --git a/gomill/gomill/sgf_grammar.py b/gomill/gomill/sgf_grammar.py new file mode 100644 index 0000000..1f6c484 --- /dev/null +++ b/gomill/gomill/sgf_grammar.py @@ -0,0 +1,513 @@ +"""Parse and serialise SGF data. + +This is intended for use with SGF FF[4]; see http://www.red-bean.com/sgf/ + +Nothing in this module is Go-specific. + +This module is encoding-agnostic: it works with 8-bit strings in an arbitrary +'ascii-compatible' encoding. + + +In the documentation below, a _property map_ is a dict mapping a PropIdent to a +nonempty list of raw property values. The keys should pass +is_valid_property_identifier(), and all values should pass +is_valid_property_value(). + +""" + +import re +import string + + +_propident_re = re.compile(r"\A[A-Z]{1,8}\Z") +_propvalue_re = re.compile(r"\A [^\\\]]* (?: \\. [^\\\]]* )* \Z", + re.VERBOSE | re.DOTALL) +_find_start_re = re.compile(r"\(\s*;") +_tokenise_re = re.compile(r""" +\s* +(?: + \[ (?P [^\\\]]* (?: \\. [^\\\]]* )* ) \] # PropValue + | + (?P [A-Z]{1,8} ) # PropIdent + | + (?P [;()] ) # delimiter +) +""", re.VERBOSE | re.DOTALL) + + +def is_valid_property_identifier(s): + """Check whether 's' is a well-formed PropIdent. + + s -- 8-bit string + + This accepts the same values as the tokeniser. + + Details: + - it doesn't permit lower-case letters (these are allowed in some ancient + SGF variants) + - it accepts at most 8 letters (there is no limit in the spec; no standard + property has more than 2) + + """ + return bool(_propident_re.search(s)) + +def is_valid_property_value(s): + """Check whether 's' is a well-formed PropValue. + + s -- 8-bit string + + This accepts the same values as the tokeniser: any string that doesn't + contain an unescaped ] or end with an unescaped \ . + + """ + return bool(_propvalue_re.search(s)) + +def tokenise(s, start_position=0): + """Tokenise a string containing SGF data. + + s -- 8-bit string + start_position -- index into 's' + + Skips leading junk. + + Returns a list of pairs of strings (token type, contents), and also the + index in 's' of the start of the unprocessed 'tail'. + + token types and contents: + I -- PropIdent: upper-case letters + V -- PropValue: raw value, without the enclosing brackets + D -- delimiter: ';', '(', or ')' + + Stops when it has seen as many closing parens as open ones, at the end of + the string, or when it first finds something it can't tokenise. + + The first two tokens are always '(' and ';' (otherwise it won't find the + start of the content). + + """ + result = [] + m = _find_start_re.search(s, start_position) + if not m: + return [], 0 + i = m.start() + depth = 0 + while True: + m = _tokenise_re.match(s, i) + if not m: + break + group = m.lastgroup + token = m.group(m.lastindex) + result.append((group, token)) + i = m.end() + if group == 'D': + if token == '(': + depth += 1 + elif token == ')': + depth -= 1 + if depth == 0: + break + return result, i + +class Coarse_game_tree(object): + """An SGF GameTree. + + This is a direct representation of the SGF parse tree. It's 'coarse' in the + sense that the objects in the tree structure represent node sequences, not + individual nodes. + + Public attributes + sequence -- nonempty list of property maps + children -- list of Coarse_game_trees + + The sequence represents the nodes before the variations. + + A raw property value is an 8-bit string containing a PropValue without its + enclosing brackets, but with backslashes and line endings left untouched + (passing is_valid_property_value()). + + """ + def __init__(self): + self.sequence = [] # must be at least one node + self.children = [] # may be empty + +def _parse_sgf_game(s, start_position): + """Common implementation for parse_sgf_game and parse_sgf_games.""" + tokens, end_position = tokenise(s, start_position) + if not tokens: + return None, None + stack = [] + game_tree = None + sequence = None + properties = None + index = 0 + try: + while True: + token_type, token = tokens[index] + index += 1 + if token_type == 'V': + raise ValueError("unexpected value") + if token_type == 'D': + if token == ';': + if sequence is None: + raise ValueError("unexpected node") + properties = {} + sequence.append(properties) + else: + if sequence is not None: + if not sequence: + raise ValueError("empty sequence") + game_tree.sequence = sequence + sequence = None + if token == '(': + stack.append(game_tree) + game_tree = Coarse_game_tree() + sequence = [] + else: + # token == ')' + variation = game_tree + game_tree = stack.pop() + if game_tree is None: + break + game_tree.children.append(variation) + properties = None + else: + # token_type == 'I' + prop_ident = token + prop_values = [] + while True: + token_type, token = tokens[index] + if token_type != 'V': + break + index += 1 + prop_values.append(token) + if not prop_values: + raise ValueError("property with no values") + try: + if prop_ident in properties: + properties[prop_ident] += prop_values + else: + properties[prop_ident] = prop_values + except TypeError: + raise ValueError("property value outside a node") + except IndexError: + raise ValueError("unexpected end of SGF data") + assert index == len(tokens) + return variation, end_position + +def parse_sgf_game(s): + """Read a single SGF game from a string, returning the parse tree. + + s -- 8-bit string + + Returns a Coarse_game_tree. + + Applies the rules for FF[4]. + + Raises ValueError if can't parse the string. + + If a property appears more than once in a node (which is not permitted by + the spec), treats it the same as a single property with multiple values. + + + Identifies the start of the SGF content by looking for '(;' (with possible + whitespace between); ignores everything preceding that. Ignores everything + following the first game. + + """ + game_tree, _ = _parse_sgf_game(s, 0) + if game_tree is None: + raise ValueError("no SGF data found") + return game_tree + +def parse_sgf_collection(s): + """Read an SGF game collection, returning the parse trees. + + s -- 8-bit string + + Returns a nonempty list of Coarse_game_trees. + + Raises ValueError if no games were found in the string. + + Raises ValueError if there is an error parsing a game. See + parse_sgf_game() for details. + + + Ignores non-SGF data before the first game, between games, and after the + final game. Identifies the start of each game in the same way as + parse_sgf_game(). + + """ + position = 0 + result = [] + while True: + try: + game_tree, position = _parse_sgf_game(s, position) + except ValueError, e: + raise ValueError("error parsing game %d: %s" % (len(result), e)) + if game_tree is None: + break + result.append(game_tree) + if not result: + raise ValueError("no SGF data found") + return result + + +def block_format(pieces, width=79): + """Concatenate strings, adding newlines. + + pieces -- iterable of strings + width -- int (default 79) + + Returns "".join(pieces), with added newlines between pieces as necessary to + avoid lines longer than 'width'. + + Leaves newlines inside 'pieces' untouched, and ignores them in its width + calculation. If a single piece is longer than 'width', it will become a + single long line in the output. + + """ + lines = [] + line = "" + for s in pieces: + if len(line) + len(s) > width: + lines.append(line) + line = "" + line += s + if line: + lines.append(line) + return "\n".join(lines) + +def serialise_game_tree(game_tree, wrap=79): + """Serialise an SGF game as a string. + + game_tree -- Coarse_game_tree + wrap -- int (default 79), or None + + Returns an 8-bit string, ending with a newline. + + If 'wrap' is not None, makes some effort to keep output lines no longer + than 'wrap'. + + """ + l = [] + to_serialise = [game_tree] + while to_serialise: + game_tree = to_serialise.pop() + if game_tree is None: + l.append(")") + continue + l.append("(") + for properties in game_tree.sequence: + l.append(";") + # Force FF to the front, largely to work around a Quarry bug which + # makes it ignore the first few bytes of the file. + for prop_ident, prop_values in sorted( + properties.iteritems(), + key=lambda (ident, _,): (-(ident=="FF"), ident)): + # Make a single string for each property, to get prettier + # block_format output. + m = [prop_ident] + for value in prop_values: + m.append("[%s]" % value) + l.append("".join(m)) + to_serialise.append(None) + to_serialise.extend(reversed(game_tree.children)) + l.append("\n") + if wrap is None: + return "".join(l) + else: + return block_format(l, wrap) + + +def make_tree(game_tree, root, node_builder, node_adder): + """Construct a node tree from a Coarse_game_tree. + + game_tree -- Coarse_game_tree + root -- node + node_builder -- function taking parameters (parent node, property map) + returning a node + node_adder -- function taking a pair (parent node, child node) + + Builds a tree of nodes corresponding to this GameTree, calling + node_builder() to make new nodes and node_adder() to add child nodes to + their parent. + + Makes no further assumptions about the node type. + + """ + to_build = [(root, game_tree, 0)] + while to_build: + node, game_tree, index = to_build.pop() + if index < len(game_tree.sequence) - 1: + child = node_builder(node, game_tree.sequence[index+1]) + node_adder(node, child) + to_build.append((child, game_tree, index+1)) + else: + node._children = [] + for child_tree in game_tree.children: + child = node_builder(node, child_tree.sequence[0]) + node_adder(node, child) + to_build.append((child, child_tree, 0)) + +def make_coarse_game_tree(root, get_children, get_properties): + """Construct a Coarse_game_tree from a node tree. + + root -- node + get_children -- function taking a node, returning a sequence of nodes + get_properties -- function taking a node, returning a property map + + Returns a Coarse_game_tree. + + Walks the node tree based at 'root' using get_children(), and uses + get_properties() to extract the raw properties. + + Makes no further assumptions about the node type. + + Doesn't check that the property maps have well-formed keys and values. + + """ + result = Coarse_game_tree() + to_serialise = [(result, root)] + while to_serialise: + game_tree, node = to_serialise.pop() + while True: + game_tree.sequence.append(get_properties(node)) + children = get_children(node) + if len(children) != 1: + break + node = children[0] + for child in children: + child_tree = Coarse_game_tree() + game_tree.children.append(child_tree) + to_serialise.append((child_tree, child)) + return result + + +def main_sequence_iter(game_tree): + """Provide the 'leftmost' complete sequence of a Coarse_game_tree. + + game_tree -- Coarse_game_tree + + Returns an iterable of property maps. + + If the game has no variations, this provides the complete game. Otherwise, + it chooses the first variation each time it has a choice. + + """ + while True: + for properties in game_tree.sequence: + yield properties + if not game_tree.children: + break + game_tree = game_tree.children[0] + + +_split_compose_re = re.compile( + r"( (?: [^\\:] | \\. )* ) :", + re.VERBOSE | re.DOTALL) + +def parse_compose(s): + """Split the parts of an SGF Compose value. + + If the value is a well-formed Compose, returns a pair of strings. + + If it isn't (ie, there is no delimiter), returns the complete string and + None. + + Interprets backslash escapes in order to find the delimiter, but leaves + backslash escapes unchanged in the returned strings. + + """ + m = _split_compose_re.match(s) + if not m: + return s, None + return m.group(1), s[m.end():] + +def compose(s1, s2): + """Construct a value of Compose value type. + + s1, s2 -- serialised form of a property value + + (This is only needed if the type of the first value permits colons.) + + """ + return s1.replace(":", "\\:") + ":" + s2 + + +_newline_re = re.compile(r"\n\r|\r\n|\n|\r") +_whitespace_table = string.maketrans("\t\f\v", " ") +_chunk_re = re.compile(r" [^\n\\]+ | [\n\\] ", re.VERBOSE) + +def simpletext_value(s): + """Convert a raw SimpleText property value to the string it represents. + + Returns an 8-bit string, in the encoding of the original SGF string. + + This interprets escape characters, and does whitespace mapping: + + - backslash followed by linebreak (LF, CR, LFCR, or CRLF) disappears + - any other linebreak is replaced by a space + - any other whitespace character is replaced by a space + - other backslashes disappear (but double-backslash -> single-backslash) + + """ + s = _newline_re.sub("\n", s) + s = s.translate(_whitespace_table) + is_escaped = False + result = [] + for chunk in _chunk_re.findall(s): + if is_escaped: + if chunk != "\n": + result.append(chunk) + is_escaped = False + elif chunk == "\\": + is_escaped = True + elif chunk == "\n": + result.append(" ") + else: + result.append(chunk) + return "".join(result) + +def text_value(s): + """Convert a raw Text property value to the string it represents. + + Returns an 8-bit string, in the encoding of the original SGF string. + + This interprets escape characters, and does whitespace mapping: + + - linebreak (LF, CR, LFCR, or CRLF) is converted to \n + - any other whitespace character is replaced by a space + - backslash followed by linebreak disappears + - other backslashes disappear (but double-backslash -> single-backslash) + + """ + s = _newline_re.sub("\n", s) + s = s.translate(_whitespace_table) + is_escaped = False + result = [] + for chunk in _chunk_re.findall(s): + if is_escaped: + if chunk != "\n": + result.append(chunk) + is_escaped = False + elif chunk == "\\": + is_escaped = True + else: + result.append(chunk) + return "".join(result) + +def escape_text(s): + """Convert a string to a raw Text property value that represents it. + + s -- 8-bit string, in the desired output encoding. + + Returns an 8-bit string which passes is_valid_property_value(). + + Normally text_value(escape_text(s)) == s, but there are the following + exceptions: + - all linebreaks are are normalised to \n + - whitespace other than line breaks is converted to a single space + + """ + return s.replace("\\", "\\\\").replace("]", "\\]") + diff --git a/gomill/gomill/sgf_moves.py b/gomill/gomill/sgf_moves.py new file mode 100644 index 0000000..26e3802 --- /dev/null +++ b/gomill/gomill/sgf_moves.py @@ -0,0 +1,99 @@ +"""Higher-level processing of moves and positions from SGF games.""" + +from gomill import boards +from gomill import sgf_properties + + +def get_setup_and_moves(sgf_game, board=None): + """Return the initial setup and the following moves from an Sgf_game. + + Returns a pair (board, plays) + + board -- boards.Board + plays -- list of pairs (colour, move) + moves are (row, col), or None for a pass. + + The board represents the position described by AB and/or AW properties + in the root node. + + The moves are from the game's 'leftmost' variation. + + Raises ValueError if this position isn't legal. + + Raises ValueError if there are any AB/AW/AE properties after the root + node. + + Doesn't check whether the moves are legal. + + If the optional 'board' parameter is provided, it must be an empty board of + the right size; the same object will be returned. + + """ + size = sgf_game.get_size() + if board is None: + board = boards.Board(size) + else: + if board.side != size: + raise ValueError("wrong board size, must be %d" % size) + if not board.is_empty(): + raise ValueError("board not empty") + root = sgf_game.get_root() + nodes = sgf_game.main_sequence_iter() + ab, aw, ae = root.get_setup_stones() + if ab or aw: + is_legal = board.apply_setup(ab, aw, ae) + if not is_legal: + raise ValueError("setup position not legal") + colour, raw = root.get_raw_move() + if colour is not None: + raise ValueError("mixed setup and moves in root node") + nodes.next() + moves = [] + for node in nodes: + if node.has_setup_stones(): + raise ValueError("setup properties after the root node") + colour, raw = node.get_raw_move() + if colour is not None: + moves.append((colour, sgf_properties.interpret_go_point(raw, size))) + return board, moves + +def set_initial_position(sgf_game, board): + """Add setup stones to an Sgf_game reflecting a board position. + + sgf_game -- Sgf_game + board -- boards.Board + + Replaces any existing setup stones in the Sgf_game's root node. + + """ + stones = {'b' : set(), 'w' : set()} + for (colour, point) in board.list_occupied_points(): + stones[colour].add(point) + sgf_game.get_root().set_setup_stones(stones['b'], stones['w']) + +def indicate_first_player(sgf_game): + """Add a PL property to the root node if appropriate. + + Looks at the first child of the root to see who the first player is, and + sets PL it isn't the expected player (ie, black normally, but white if + there is a handicap), or if there are non-handicap setup stones. + + """ + root = sgf_game.get_root() + first_player, move = root[0].get_move() + if first_player is None: + return + has_handicap = root.has_property("HA") + if root.has_property("AW"): + specify_pl = True + elif root.has_property("AB") and not has_handicap: + specify_pl = True + elif not has_handicap and first_player == 'w': + specify_pl = True + elif has_handicap and first_player == 'b': + specify_pl = True + else: + specify_pl = False + if specify_pl: + root.set('PL', first_player) + diff --git a/gomill/gomill/sgf_properties.py b/gomill/gomill/sgf_properties.py new file mode 100644 index 0000000..a107527 --- /dev/null +++ b/gomill/gomill/sgf_properties.py @@ -0,0 +1,730 @@ +"""Interpret SGF property values. + +This is intended for use with SGF FF[4]; see http://www.red-bean.com/sgf/ + +This supports all general properties and Go-specific properties, but not +properties for other games. Point, Move and Stone values are interpreted as Go +points. + +""" + +import codecs + +from gomill import sgf_grammar +from gomill.utils import isinf, isnan + +def normalise_charset_name(s): + """Convert an encoding name to the form implied in the SGF spec. + + In particular, normalises to 'ISO-8859-1' and 'UTF-8'. + + Raises LookupError if the encoding name isn't known to Python. + + """ + return (codecs.lookup(s).name.replace("_", "-").upper() + .replace("ISO8859", "ISO-8859")) + + +def interpret_go_point(s, size): + """Convert a raw SGF Go Point, Move, or Stone value to coordinates. + + s -- 8-bit string + size -- board size (int) + + Returns a pair (row, col), or None for a pass. + + Raises ValueError if the string is malformed or the coordinates are out of + range. + + Only supports board sizes up to 26. + + The returned coordinates are in the GTP coordinate system (as in the rest + of gomill), where (0, 0) is the lower left. + + """ + if s == "" or (s == "tt" and size <= 19): + return None + # May propagate ValueError + col_s, row_s = s + col = ord(col_s) - 97 # 97 == ord("a") + row = size - ord(row_s) + 96 + if not ((0 <= col < size) and (0 <= row < size)): + raise ValueError + return row, col + +def serialise_go_point(move, size): + """Serialise a Go Point, Move, or Stone value. + + move -- pair (row, col), or None for a pass + + Returns an 8-bit string. + + Only supports board sizes up to 26. + + The move coordinates are in the GTP coordinate system (as in the rest of + gomill), where (0, 0) is the lower left. + + """ + if not 1 <= size <= 26: + raise ValueError + if move is None: + # Prefer 'tt' where possible, for the sake of older code + if size <= 19: + return "tt" + else: + return "" + row, col = move + if not ((0 <= col < size) and (0 <= row < size)): + raise ValueError + col_s = "abcdefghijklmnopqrstuvwxy"[col] + row_s = "abcdefghijklmnopqrstuvwxy"[size - row - 1] + return col_s + row_s + + +class _Context(object): + def __init__(self, size, encoding): + self.size = size + self.encoding = encoding + +def interpret_none(s, context=None): + """Convert a raw None value to a boolean. + + That is, unconditionally returns True. + + """ + return True + +def serialise_none(b, context=None): + """Serialise a None value. + + Ignores its parameter. + + """ + return "" + + +def interpret_number(s, context=None): + """Convert a raw Number value to the integer it represents. + + This is a little more lenient than the SGF spec: it permits leading and + trailing spaces, and spaces between the sign and the numerals. + + """ + return int(s, 10) + +def serialise_number(i, context=None): + """Serialise a Number value. + + i -- integer + + """ + return "%d" % i + + +def interpret_real(s, context=None): + """Convert a raw Real value to the float it represents. + + This is more lenient than the SGF spec: it accepts strings accepted as a + float by the platform libc. It rejects infinities and NaNs. + + """ + result = float(s) + if isinf(result): + raise ValueError("infinite") + if isnan(result): + raise ValueError("not a number") + return result + +def serialise_real(f, context=None): + """Serialise a Real value. + + f -- real number (int or float) + + If the absolute value is too small to conveniently express as a decimal, + returns "0" (this currently happens if abs(f) is less than 0.0001). + + """ + f = float(f) + try: + i = int(f) + except OverflowError: + # infinity + raise ValueError + if f == i: + # avoid trailing '.0'; also avoid scientific notation for large numbers + return str(i) + s = repr(f) + if 'e-' in s: + return "0" + return s + + +def interpret_double(s, context=None): + """Convert a raw Double value to an integer. + + Returns 1 or 2 (unknown values are treated as 1). + + """ + if s.strip() == "2": + return 2 + else: + return 1 + +def serialise_double(i, context=None): + """Serialise a Double value. + + i -- integer (1 or 2) + + (unknown values are treated as 1) + + """ + if i == 2: + return "2" + return "1" + + +def interpret_colour(s, context=None): + """Convert a raw Color value to a gomill colour. + + Returns 'b' or 'w'. + + """ + colour = s.lower() + if colour not in ('b', 'w'): + raise ValueError + return colour + +def serialise_colour(colour, context=None): + """Serialise a Colour value. + + colour -- 'b' or 'w' + + """ + if colour not in ('b', 'w'): + raise ValueError + return colour.upper() + + +def _transcode(s, encoding): + """Common implementation for interpret_text and interpret_simpletext.""" + # If encoding is UTF-8, we don't need to transcode, but we still want to + # report an error if it's not property encoded. + u = s.decode(encoding) + if encoding == "UTF-8": + return s + else: + return u.encode("utf-8") + +def interpret_simpletext(s, context): + """Convert a raw SimpleText value to a string. + + See sgf_grammar.simpletext_value() for details. + + s -- raw value + + Returns an 8-bit utf-8 string. + + """ + return _transcode(sgf_grammar.simpletext_value(s), context.encoding) + +def serialise_simpletext(s, context): + """Serialise a SimpleText value. + + See sgf_grammar.escape_text() for details. + + s -- 8-bit utf-8 string + + """ + if context.encoding != "UTF-8": + s = s.decode("utf-8").encode(context.encoding) + return sgf_grammar.escape_text(s) + + +def interpret_text(s, context): + """Convert a raw Text value to a string. + + See sgf_grammar.text_value() for details. + + s -- raw value + + Returns an 8-bit utf-8 string. + + """ + return _transcode(sgf_grammar.text_value(s), context.encoding) + +def serialise_text(s, context): + """Serialise a Text value. + + See sgf_grammar.escape_text() for details. + + s -- 8-bit utf-8 string + + """ + if context.encoding != "UTF-8": + s = s.decode("utf-8").encode(context.encoding) + return sgf_grammar.escape_text(s) + + + +def interpret_point(s, context): + """Convert a raw SGF Point or Stone value to coordinates. + + See interpret_go_point() above for details. + + Returns a pair (row, col). + + """ + result = interpret_go_point(s, context.size) + if result is None: + raise ValueError + return result + +def serialise_point(point, context): + """Serialise a Point or Stone value. + + point -- pair (row, col) + + See serialise_go_point() above for details. + + """ + if point is None: + raise ValueError + return serialise_go_point(point, context.size) + + +def interpret_move(s, context): + """Convert a raw SGF Move value to coordinates. + + See interpret_go_point() above for details. + + Returns a pair (row, col), or None for a pass. + + """ + return interpret_go_point(s, context.size) + +def serialise_move(move, context): + """Serialise a Move value. + + move -- pair (row, col), or None for a pass + + See serialise_go_point() above for details. + + """ + return serialise_go_point(move, context.size) + + +def interpret_point_list(values, context): + """Convert a raw SGF list of Points to a set of coordinates. + + values -- list of strings + + Returns a set of pairs (row, col). + + If 'values' is empty, returns an empty set. + + This interprets compressed point lists. + + Doesn't complain if there is overlap, or if a single point is specified as + a 1x1 rectangle. + + Raises ValueError if the data is otherwise malformed. + + """ + result = set() + for s in values: + # No need to use parse_compose(), as \: would always be an error. + p1, is_rectangle, p2 = s.partition(":") + if is_rectangle: + top, left = interpret_point(p1, context) + bottom, right = interpret_point(p2, context) + if not (bottom <= top and left <= right): + raise ValueError + for row in xrange(bottom, top+1): + for col in xrange(left, right+1): + result.add((row, col)) + else: + pt = interpret_point(p1, context) + result.add(pt) + return result + +def serialise_point_list(points, context): + """Serialise a list of Points, Moves, or Stones. + + points -- iterable of pairs (row, col) + + Returns a list of strings. + + If 'points' is empty, returns an empty list. + + Doesn't produce a compressed point list. + + """ + result = [serialise_point(point, context) for point in points] + result.sort() + return result + + +def interpret_AP(s, context): + """Interpret an AP (application) property value. + + Returns a pair of strings (name, version number) + + Permits the version number to be missing (which is forbidden by the SGF + spec), in which case the second returned value is an empty string. + + """ + application, version = sgf_grammar.parse_compose(s) + if version is None: + version = "" + return (interpret_simpletext(application, context), + interpret_simpletext(version, context)) + +def serialise_AP(value, context): + """Serialise an AP (application) property value. + + value -- pair (application, version) + application -- string + version -- string + + Note this takes a single parameter (which is a pair). + + """ + application, version = value + return sgf_grammar.compose(serialise_simpletext(application, context), + serialise_simpletext(version, context)) + + +def interpret_ARLN_list(values, context): + """Interpret an AR (arrow) or LN (line) property value. + + Returns a list of pairs (point, point), where point is a pair (row, col) + + """ + result = [] + for s in values: + p1, p2 = sgf_grammar.parse_compose(s) + result.append((interpret_point(p1, context), + interpret_point(p2, context))) + return result + +def serialise_ARLN_list(values, context): + """Serialise an AR (arrow) or LN (line) property value. + + values -- list of pairs (point, point), where point is a pair (row, col) + + """ + return ["%s:%s" % (serialise_point(p1, context), + serialise_point(p2, context)) + for p1, p2 in values] + + +def interpret_FG(s, context): + """Interpret an FG (figure) property value. + + Returns a pair (flags, string), or None. + + flags is an integer; see http://www.red-bean.com/sgf/properties.html#FG + + """ + if s == "": + return None + flags, name = sgf_grammar.parse_compose(s) + return int(flags), interpret_simpletext(name, context) + +def serialise_FG(value, context): + """Serialise an FG (figure) property value. + + value -- pair (flags, name), or None + flags -- int + name -- string + + Use serialise_FG(None) to produce an empty value. + + """ + if value is None: + return "" + flags, name = value + return "%d:%s" % (flags, serialise_simpletext(name, context)) + + +def interpret_LB_list(values, context): + """Interpret an LB (label) property value. + + Returns a list of pairs ((row, col), string). + + """ + result = [] + for s in values: + point, label = sgf_grammar.parse_compose(s) + result.append((interpret_point(point, context), + interpret_simpletext(label, context))) + return result + +def serialise_LB_list(values, context): + """Serialise an LB (label) property value. + + values -- list of pairs ((row, col), string) + + """ + return ["%s:%s" % (serialise_point(point, context), + serialise_simpletext(text, context)) + for point, text in values] + + +class Property_type(object): + """Description of a property type.""" + def __init__(self, interpreter, serialiser, uses_list, + allows_empty_list=False): + self.interpreter = interpreter + self.serialiser = serialiser + self.uses_list = bool(uses_list) + self.allows_empty_list = bool(allows_empty_list) + +def _make_property_type(type_name, allows_empty_list=False): + return Property_type( + globals()["interpret_" + type_name], + globals()["serialise_" + type_name], + uses_list=(type_name.endswith("_list")), + allows_empty_list=allows_empty_list) + +_property_types_by_name = { + 'none' : _make_property_type('none'), + 'number' : _make_property_type('number'), + 'real' : _make_property_type('real'), + 'double' : _make_property_type('double'), + 'colour' : _make_property_type('colour'), + 'simpletext' : _make_property_type('simpletext'), + 'text' : _make_property_type('text'), + 'point' : _make_property_type('point'), + 'move' : _make_property_type('move'), + 'point_list' : _make_property_type('point_list'), + 'point_elist' : _make_property_type('point_list', allows_empty_list=True), + 'stone_list' : _make_property_type('point_list'), + 'AP' : _make_property_type('AP'), + 'ARLN_list' : _make_property_type('ARLN_list'), + 'FG' : _make_property_type('FG'), + 'LB_list' : _make_property_type('LB_list'), +} + +P = _property_types_by_name + +_property_types_by_ident = { + 'AB' : P['stone_list'], # setup Add Black + 'AE' : P['point_list'], # setup Add Empty + 'AN' : P['simpletext'], # game-info Annotation + 'AP' : P['AP'], # root Application + 'AR' : P['ARLN_list'], # - Arrow + 'AW' : P['stone_list'], # setup Add White + 'B' : P['move'], # move Black + 'BL' : P['real'], # move Black time left + 'BM' : P['double'], # move Bad move + 'BR' : P['simpletext'], # game-info Black rank + 'BT' : P['simpletext'], # game-info Black team + 'C' : P['text'], # - Comment + 'CA' : P['simpletext'], # root Charset + 'CP' : P['simpletext'], # game-info Copyright + 'CR' : P['point_list'], # - Circle + 'DD' : P['point_elist'], # - [inherit] Dim points + 'DM' : P['double'], # - Even position + 'DO' : P['none'], # move Doubtful + 'DT' : P['simpletext'], # game-info Date + 'EV' : P['simpletext'], # game-info Event + 'FF' : P['number'], # root Fileformat + 'FG' : P['FG'], # - Figure + 'GB' : P['double'], # - Good for Black + 'GC' : P['text'], # game-info Game comment + 'GM' : P['number'], # root Game + 'GN' : P['simpletext'], # game-info Game name + 'GW' : P['double'], # - Good for White + 'HA' : P['number'], # game-info Handicap + 'HO' : P['double'], # - Hotspot + 'IT' : P['none'], # move Interesting + 'KM' : P['real'], # game-info Komi + 'KO' : P['none'], # move Ko + 'LB' : P['LB_list'], # - Label + 'LN' : P['ARLN_list'], # - Line + 'MA' : P['point_list'], # - Mark + 'MN' : P['number'], # move set move number + 'N' : P['simpletext'], # - Nodename + 'OB' : P['number'], # move OtStones Black + 'ON' : P['simpletext'], # game-info Opening + 'OT' : P['simpletext'], # game-info Overtime + 'OW' : P['number'], # move OtStones White + 'PB' : P['simpletext'], # game-info Player Black + 'PC' : P['simpletext'], # game-info Place + 'PL' : P['colour'], # setup Player to play + 'PM' : P['number'], # - [inherit] Print move mode + 'PW' : P['simpletext'], # game-info Player White + 'RE' : P['simpletext'], # game-info Result + 'RO' : P['simpletext'], # game-info Round + 'RU' : P['simpletext'], # game-info Rules + 'SL' : P['point_list'], # - Selected + 'SO' : P['simpletext'], # game-info Source + 'SQ' : P['point_list'], # - Square + 'ST' : P['number'], # root Style + 'SZ' : P['number'], # root Size + 'TB' : P['point_elist'], # - Territory Black + 'TE' : P['double'], # move Tesuji + 'TM' : P['real'], # game-info Timelimit + 'TR' : P['point_list'], # - Triangle + 'TW' : P['point_elist'], # - Territory White + 'UC' : P['double'], # - Unclear pos + 'US' : P['simpletext'], # game-info User + 'V' : P['real'], # - Value + 'VW' : P['point_elist'], # - [inherit] View + 'W' : P['move'], # move White + 'WL' : P['real'], # move White time left + 'WR' : P['simpletext'], # game-info White rank + 'WT' : P['simpletext'], # game-info White team +} +_text_property_type = P['text'] + +del P + + +class Presenter(_Context): + """Convert property values between Python and SGF-string representations. + + Instantiate with: + size -- board size (int) + encoding -- encoding for the SGF strings + + Public attributes (treat as read-only): + size -- int + encoding -- string (normalised form) + + See the _property_types_by_ident table above for a list of properties + initially known, and their types. + + Initially, treats unknown (private) properties as if they had type Text. + + """ + + def __init__(self, size, encoding): + try: + encoding = normalise_charset_name(encoding) + except LookupError: + raise ValueError("unknown encoding: %s" % encoding) + _Context.__init__(self, size, encoding) + self.property_types_by_ident = _property_types_by_ident.copy() + self.default_property_type = _text_property_type + + def get_property_type(self, identifier): + """Return the Property_type for the specified PropIdent. + + Rasies KeyError if the property is unknown. + + """ + return self.property_types_by_ident[identifier] + + def register_property(self, identifier, property_type): + """Specify the Property_type for a PropIdent.""" + self.property_types_by_ident[identifier] = property_type + + def deregister_property(self, identifier): + """Forget the type for the specified PropIdent.""" + del self.property_types_by_ident[identifier] + + def set_private_property_type(self, property_type): + """Specify the Property_type to use for unknown properties. + + Pass property_type = None to make unknown properties raise an error. + + """ + self.default_property_type = property_type + + def _get_effective_property_type(self, identifier): + try: + return self.property_types_by_ident[identifier] + except KeyError: + result = self.default_property_type + if result is None: + raise ValueError("unknown property") + return result + + def interpret_as_type(self, property_type, raw_values): + """Variant of interpret() for explicitly specified type. + + property_type -- Property_type + + """ + if not raw_values: + raise ValueError("no raw values") + if property_type.uses_list: + if raw_values == [""]: + raw = [] + else: + raw = raw_values + else: + if len(raw_values) > 1: + raise ValueError("multiple values") + raw = raw_values[0] + return property_type.interpreter(raw, self) + + def interpret(self, identifier, raw_values): + """Return a Python representation of a property value. + + identifier -- PropIdent + raw_values -- nonempty list of 8-bit strings in the presenter's encoding + + See the interpret_... functions above for details of how values are + represented as Python types. + + Raises ValueError if it cannot interpret the value. + + Note that in some cases the interpret_... functions accept values which + are not strictly permitted by the specification. + + elist handling: if the property's value type is a list type and + 'raw_values' is a list containing a single empty string, passes an + empty list to the interpret_... function (that is, this function treats + all lists like elists). + + Doesn't enforce range restrictions on values with type Number. + + """ + return self.interpret_as_type( + self._get_effective_property_type(identifier), raw_values) + + def serialise_as_type(self, property_type, value): + """Variant of serialise() for explicitly specified type. + + property_type -- Property_type + + """ + serialised = property_type.serialiser(value, self) + if property_type.uses_list: + if serialised == []: + if property_type.allows_empty_list: + return [""] + else: + raise ValueError("empty list") + return serialised + else: + return [serialised] + + def serialise(self, identifier, value): + """Serialise a Python representation of a property value. + + identifier -- PropIdent + value -- corresponding Python value + + Returns a nonempty list of 8-bit strings in the presenter's encoding, + suitable for use as raw PropValues. + + See the serialise_... functions above for details of the acceptable + values for each type. + + elist handling: if the property's value type is an elist type and the + serialise_... function returns an empty list, this returns a list + containing a single empty string. + + Raises ValueError if it cannot serialise the value. + + In general, the serialise_... functions try not to produce an invalid + result, but do not try to prevent garbage input happening to produce a + valid result. + + """ + return self.serialise_as_type( + self._get_effective_property_type(identifier), value) diff --git a/gomill/gomill/terminal_input.py b/gomill/gomill/terminal_input.py new file mode 100644 index 0000000..1ee0641 --- /dev/null +++ b/gomill/gomill/terminal_input.py @@ -0,0 +1,83 @@ +"""Support for non-blocking terminal input.""" + +import os + +try: + import termios +except ImportError: + termios = None + +class Terminal_reader(object): + """Check for input on the controlling terminal.""" + + def __init__(self): + self.enabled = True + self.tty = None + + def is_enabled(self): + return self.enabled + + def disable(self): + self.enabled = False + + def initialise(self): + if not self.enabled: + return + if termios is None: + self.enabled = False + return + try: + self.tty = open("/dev/tty", "w+") + os.tcgetpgrp(self.tty.fileno()) + self.clean_tcattr = termios.tcgetattr(self.tty) + iflag, oflag, cflag, lflag, ispeed, ospeed, cc = self.clean_tcattr + new_lflag = lflag & (0xffffffff ^ termios.ICANON) + new_cc = cc[:] + new_cc[termios.VMIN] = 0 + self.cbreak_tcattr = [ + iflag, oflag, cflag, new_lflag, ispeed, ospeed, new_cc] + except Exception: + self.enabled = False + return + + def close(self): + if self.tty is not None: + self.tty.close() + self.tty = None + + def stop_was_requested(self): + """Check whether a 'keyboard stop' instruction has been sent. + + Returns true if ^X has been sent on the controlling terminal. + + Consumes all available input on /dev/tty. + + """ + if not self.enabled: + return False + # Don't try to read the terminal if we're in the background. + # There's a race here, if we're backgrounded just after this check, but + # I don't see a clean way to avoid it. + if os.tcgetpgrp(self.tty.fileno()) != os.getpid(): + return False + try: + termios.tcsetattr(self.tty, termios.TCSANOW, self.cbreak_tcattr) + except EnvironmentError: + return False + try: + seen_ctrl_x = False + while True: + c = os.read(self.tty.fileno(), 1) + if not c: + break + if c == "\x18": + seen_ctrl_x = True + except EnvironmentError: + seen_ctrl_x = False + finally: + termios.tcsetattr(self.tty, termios.TCSANOW, self.clean_tcattr) + return seen_ctrl_x + + def acknowledge(self): + """Leave an acknowledgement on the controlling terminal.""" + self.tty.write("\rCtrl-X received; halting\n") diff --git a/gomill/gomill/tournament_results.py b/gomill/gomill/tournament_results.py new file mode 100644 index 0000000..82d5718 --- /dev/null +++ b/gomill/gomill/tournament_results.py @@ -0,0 +1,311 @@ +"""Retrieving and reporting on tournament results.""" + +from __future__ import division + +from gomill import ascii_tables +from gomill.utils import format_float, format_percent +from gomill.common import colour_name + +class Matchup_description(object): + """Description of a matchup (pairing of two players). + + Public attributes: + id -- matchup id (very short string) + player_1 -- player code (identifier-like string) + player_2 -- player code (identifier-like string) + name -- string (eg 'xxx v yyy') + board_size -- int + komi -- float + alternating -- bool + handicap -- int or None + handicap_style -- 'fixed' or 'free' + move_limit -- int + scorer -- 'internal' or 'players' + number_of_games -- int or None + + If alternating is False, player_1 plays black and player_2 plays white; + otherwise they alternate. + + player_1 and player_2 are always different. + + """ + def describe_details(self): + """Return a text description of game settings. + + This covers the most important game settings which can't be observed + in the results table (board size, handicap, and komi). + + """ + s = "board size: %s " % self.board_size + if self.handicap is not None: + s += "handicap: %s (%s) " % ( + self.handicap, self.handicap_style) + s += "komi: %s" % self.komi + return s + + +class Tournament_results(object): + """Provide access to results of a single tournament. + + The tournament results are catalogued in terms of 'matchups', with each + matchup corresponding to a series of games which have the same players and + settings. Each matchup has an id, which is a short string. + + """ + def __init__(self, matchup_list, results): + self.matchup_list = matchup_list + self.results = results + self.matchups = dict((m.id, m) for m in matchup_list) + + def get_matchup_ids(self): + """Return a list of all matchup ids, in definition order.""" + return [m.id for m in self.matchup_list] + + def get_matchup(self, matchup_id): + """Describe the matchup with the specified id. + + Returns a Matchup_description (which should be treated as read-only). + + """ + return self.matchups[matchup_id] + + def get_matchups(self): + """Return a map matchup id -> Matchup_description.""" + return self.matchups.copy() + + def get_matchup_results(self, matchup_id): + """Return the results for the specified matchup. + + Returns a list of gtp_games.Game_results (in unspecified order). + + The Game_results all have game_id set. + + """ + return self.results[matchup_id][:] + + def get_matchup_stats(self, matchup_id): + """Return statistics for the specified matchup. + + Returns a Matchup_stats object. + + """ + matchup = self.matchups[matchup_id] + ms = Matchup_stats(self.results[matchup_id], + matchup.player_1, matchup.player_2) + ms.calculate_colour_breakdown() + ms.calculate_time_stats() + return ms + + +class Matchup_stats(object): + """Result statistics for games between a pair of players. + + Instantiate with + results -- list of gtp_games.Game_results + player_1 -- player code + player_2 -- player code + The game results should all be for games between player_1 and player_2. + + Public attributes: + player_1 -- player code + player_2 -- player code + total -- int (number of games) + wins_1 -- float (score) + wins_2 -- float (score) + forfeits_1 -- int (number of games) + forfeits_2 -- int (number of games) + unknown -- int (number of games) + + scores are multiples of 0.5 (as there may be jigos). + + """ + def __init__(self, results, player_1, player_2): + self._results = results + self.player_1 = player_1 + self.player_2 = player_2 + + self.total = len(results) + + js = self._jigo_score = 0.5 * sum(r.is_jigo for r in results) + self.unknown = sum(r.winning_player is None and not r.is_jigo + for r in results) + + self.wins_1 = sum(r.winning_player == player_1 for r in results) + js + self.wins_2 = sum(r.winning_player == player_2 for r in results) + js + + self.forfeits_1 = sum(r.winning_player == player_2 and r.is_forfeit + for r in results) + self.forfeits_2 = sum(r.winning_player == player_1 and r.is_forfeit + for r in results) + + def calculate_colour_breakdown(self): + """Calculate futher statistics, broken down by colour played. + + Sets the following additional attributes: + + played_1b -- int (number of games) + played_1w -- int (number of games) + played_2b -- int (number of games) + played_y2 -- int (number of games) + alternating -- bool + when alternating is true => + wins_b -- float (score) + wins_w -- float (score) + wins_1b -- float (score) + wins_1w -- float (score) + wins_2b -- float (score) + wins_2w -- float (score) + else => + colour_1 -- 'b' or 'w' + colour_2 -- 'b' or 'w' + + """ + results = self._results + player_1 = self.player_1 + player_2 = self.player_2 + js = self._jigo_score + + self.played_1b = sum(r.player_b == player_1 for r in results) + self.played_1w = sum(r.player_w == player_1 for r in results) + self.played_2b = sum(r.player_b == player_2 for r in results) + self.played_y2 = sum(r.player_w == player_2 for r in results) + + if self.played_1w == 0 and self.played_2b == 0: + self.alternating = False + self.colour_1 = 'b' + self.colour_2 = 'w' + elif self.played_1b == 0 and self.played_y2 == 0: + self.alternating = False + self.colour_1 = 'w' + self.colour_2 = 'b' + else: + self.alternating = True + self.wins_b = sum(r.winning_colour == 'b' for r in results) + js + self.wins_w = sum(r.winning_colour == 'w' for r in results) + js + self.wins_1b = sum( + r.winning_player == player_1 and r.winning_colour == 'b' + for r in results) + js + self.wins_1w = sum( + r.winning_player == player_1 and r.winning_colour == 'w' + for r in results) + js + self.wins_2b = sum( + r.winning_player == player_2 and r.winning_colour == 'b' + for r in results) + js + self.wins_2w = sum( + r.winning_player == player_2 and r.winning_colour == 'w' + for r in results) + js + + def calculate_time_stats(self): + """Calculate CPU time statistics. + + average_time_1 -- float or None + average_time_2 -- float or None + + """ + player_1 = self.player_1 + player_2 = self.player_2 + times_1 = [r.cpu_times[player_1] for r in self._results] + known_times_1 = [t for t in times_1 if t is not None and t != '?'] + times_2 = [r.cpu_times[player_2] for r in self._results] + known_times_2 = [t for t in times_2 if t is not None and t != '?'] + if known_times_1: + self.average_time_1 = sum(known_times_1) / len(known_times_1) + else: + self.average_time_1 = None + if known_times_2: + self.average_time_2 = sum(known_times_2) / len(known_times_2) + else: + self.average_time_2 = None + + +def make_matchup_stats_table(ms): + """Produce an ascii table showing matchup statistics. + + ms -- Matchup_stats (with all statistics set) + + returns an ascii_tables.Table + + """ + ff = format_float + pct = format_percent + + t = ascii_tables.Table(row_count=3) + t.add_heading("") # player name + i = t.add_column(align='left', right_padding=3) + t.set_column_values(i, [ms.player_1, ms.player_2]) + + t.add_heading("wins") + i = t.add_column(align='right') + t.set_column_values(i, [ff(ms.wins_1), ff(ms.wins_2)]) + + t.add_heading("") # overall pct + i = t.add_column(align='right') + t.set_column_values(i, [pct(ms.wins_1, ms.total), + pct(ms.wins_2, ms.total)]) + + if ms.alternating: + t.columns[i].right_padding = 7 + t.add_heading("black", span=2) + i = t.add_column(align='left') + t.set_column_values(i, [ff(ms.wins_1b), ff(ms.wins_2b), ff(ms.wins_b)]) + i = t.add_column(align='right', right_padding=5) + t.set_column_values(i, [pct(ms.wins_1b, ms.played_1b), + pct(ms.wins_2b, ms.played_2b), + pct(ms.wins_b, ms.total)]) + + t.add_heading("white", span=2) + i = t.add_column(align='left') + t.set_column_values(i, [ff(ms.wins_1w), ff(ms.wins_2w), ff(ms.wins_w)]) + i = t.add_column(align='right', right_padding=3) + t.set_column_values(i, [pct(ms.wins_1w, ms.played_1w), + pct(ms.wins_2w, ms.played_y2), + pct(ms.wins_w, ms.total)]) + else: + t.columns[i].right_padding = 3 + t.add_heading("") + i = t.add_column(align='left') + t.set_column_values(i, ["(%s)" % colour_name(ms.colour_1), + "(%s)" % colour_name(ms.colour_2)]) + + if ms.forfeits_1 or ms.forfeits_2: + t.add_heading("forfeits") + i = t.add_column(align='right') + t.set_column_values(i, [ms.forfeits_1, ms.forfeits_2]) + + if ms.average_time_1 or ms.average_time_2: + if ms.average_time_1 is not None: + avg_time_1_s = "%7.2f" % ms.average_time_1 + else: + avg_time_1_s = " ----" + if ms.average_time_2 is not None: + avg_time_2_s = "%7.2f" % ms.average_time_2 + else: + avg_time_2_s = " ----" + t.add_heading("avg cpu") + i = t.add_column(align='right', right_padding=2) + t.set_column_values(i, [avg_time_1_s, avg_time_2_s]) + + return t + +def write_matchup_summary(out, matchup, ms): + """Write a summary block for the specified matchup to 'out'. + + matchup -- Matchup_description + ms -- Matchup_stats (with all statistics set) + + """ + def p(s): + print >>out, s + + if matchup.number_of_games is None: + played_s = "%d" % ms.total + else: + played_s = "%d/%d" % (ms.total, matchup.number_of_games) + p("%s (%s games)" % (matchup.name, played_s)) + if ms.unknown > 0: + p("unknown results: %d %s" % + (ms.unknown, format_percent(ms.unknown, ms.total))) + + p(matchup.describe_details()) + p("\n".join(make_matchup_stats_table(ms).render())) + diff --git a/gomill/gomill/tournaments.py b/gomill/gomill/tournaments.py new file mode 100644 index 0000000..7b63a65 --- /dev/null +++ b/gomill/gomill/tournaments.py @@ -0,0 +1,317 @@ +"""Common code for all tournament types.""" + +from collections import defaultdict + +from gomill import game_jobs +from gomill import competition_schedulers +from gomill import tournament_results +from gomill import competitions +from gomill.competitions import ( + Competition, NoGameAvailable, CompetitionError, ControlFileError) +from gomill.settings import * +from gomill.utils import format_percent + +# These all appear as Matchup_description attributes +matchup_settings = competitions.game_settings + [ + Setting('alternating', interpret_bool, default=False), + Setting('number_of_games', allow_none(interpret_int), default=None), + ] + + +class Matchup(tournament_results.Matchup_description): + """Internal description of a matchup from the configuration file. + + See tournament_results.Matchup_description for main public attributes. + + Additional attributes: + event_description -- string to show as sgf event + + Instantiate with + matchup_id -- identifier + player_1 -- player code + player_2 -- player code + parameters -- dict matchup setting name -> value + name -- utf-8 string, or None + event_code -- identifier + + If a matchup setting is missing from 'parameters', the setting's default + value is used (or, if there is no default value, ValueError is raised). + + 'event_code' is used for the sgf event description (combined with 'name' + if available). + + Instantiation raises ControlFileError if the handicap settings aren't + permitted. + + """ + def __init__(self, matchup_id, player_1, player_2, parameters, + name, event_code): + self.id = matchup_id + self.player_1 = player_1 + self.player_2 = player_2 + + for setting in matchup_settings: + try: + v = parameters[setting.name] + except KeyError: + try: + v = setting.get_default() + except KeyError: + raise ValueError("'%s' not specified" % setting.name) + setattr(self, setting.name, v) + + competitions.validate_handicap( + self.handicap, self.handicap_style, self.board_size) + + if name is None: + name = "%s v %s" % (self.player_1, self.player_2) + event_description = event_code + else: + event_description = "%s (%s)" % (event_code, name) + self.name = name + self.event_description = event_description + self._game_id_template = ("%s_" + + competitions.leading_zero_template(self.number_of_games)) + + def make_game_id(self, game_number): + return self._game_id_template % (self.id, game_number) + + +class Ghost_matchup(object): + """Dummy Matchup object for matchups which have gone from the control file. + + This is used if the matchup appears in results. + + It has to be a good enough imitation to keep write_matchup_summary() happy. + + """ + def __init__(self, matchup_id, player_1, player_2): + self.id = matchup_id + self.player_1 = player_1 + self.player_2 = player_2 + self.name = "%s v %s" % (player_1, player_2) + self.number_of_games = None + + def describe_details(self): + return "?? (missing from control file)" + + +class Tournament(Competition): + """A Competition based on a number of matchups. + + """ + def __init__(self, competition_code, **kwargs): + Competition.__init__(self, competition_code, **kwargs) + self.working_matchups = set() + self.probationary_matchups = set() + + def make_matchup(self, matchup_id, player_1, player_2, parameters, + name=None): + """Make a Matchup from the various parameters. + + Raises ControlFileError if any required parameters are missing. + + See Matchup.__init__ for details. + + """ + try: + return Matchup(matchup_id, player_1, player_2, parameters, name, + event_code=self.competition_code) + except ValueError, e: + raise ControlFileError(str(e)) + + + # State attributes (*: in persistent state): + # *results -- map matchup id -> list of Game_results + # *scheduler -- Group_scheduler (group codes are matchup ids) + # *engine_names -- map player code -> string + # *engine_descriptions -- map player code -> string + # working_matchups -- set of matchup ids + # (matchups which have successfully completed a game in this run) + # probationary_matchups -- set of matchup ids + # (matchups which failed to complete their last game) + # ghost_matchups -- map matchup id -> Ghost_matchup + # (matchups which have been removed from the control file) + + def _check_results(self): + """Check that the current results are consistent with the control file. + + This is run when reloading state. + + Raises CompetitionError if they're not. + + (In general, control file changes are permitted. The only thing we + reject is results for a currently-defined matchup whose players aren't + correct.) + + """ + # We guarantee that results for a given matchup always have consistent + # players, so we need only check the first result. + for matchup in self.matchup_list: + results = self.results[matchup.id] + if not results: + continue + result = results[0] + seen_players = sorted(result.players.itervalues()) + expected_players = sorted((matchup.player_1, matchup.player_2)) + if seen_players != expected_players: + raise CompetitionError( + "existing results for matchup %s " + "are inconsistent with control file:\n" + "result players are %s;\n" + "control file players are %s" % + (matchup.id, + ",".join(seen_players), ",".join(expected_players))) + + def _set_ghost_matchups(self): + self.ghost_matchups = {} + live = set(self.matchups) + for matchup_id, results in self.results.iteritems(): + if matchup_id in live: + continue + result = results[0] + # player_1 and player_2 might not be the right way round, but it + # doesn't matter. + self.ghost_matchups[matchup_id] = Ghost_matchup( + matchup_id, result.player_b, result.player_w) + + def _set_scheduler_groups(self): + self.scheduler.set_groups( + [(m.id, m.number_of_games) for m in self.matchup_list] + + [(id, 0) for id in self.ghost_matchups]) + + def set_clean_status(self): + self.results = defaultdict(list) + self.engine_names = {} + self.engine_descriptions = {} + self.scheduler = competition_schedulers.Group_scheduler() + self.ghost_matchups = {} + self._set_scheduler_groups() + + def get_status(self): + return { + 'results' : self.results, + 'scheduler' : self.scheduler, + 'engine_names' : self.engine_names, + 'engine_descriptions' : self.engine_descriptions, + } + + def set_status(self, status): + self.results = status['results'] + self._check_results() + self._set_ghost_matchups() + self.scheduler = status['scheduler'] + self._set_scheduler_groups() + self.scheduler.rollback() + self.engine_names = status['engine_names'] + self.engine_descriptions = status['engine_descriptions'] + + + def get_game(self): + matchup_id, game_number = self.scheduler.issue() + if matchup_id is None: + return NoGameAvailable + matchup = self.matchups[matchup_id] + if matchup.alternating and (game_number % 2): + player_b, player_w = matchup.player_2, matchup.player_1 + else: + player_b, player_w = matchup.player_1, matchup.player_2 + game_id = matchup.make_game_id(game_number) + + job = game_jobs.Game_job() + job.game_id = game_id + job.game_data = (matchup_id, game_number) + job.player_b = self.players[player_b] + job.player_w = self.players[player_w] + job.board_size = matchup.board_size + job.komi = matchup.komi + job.move_limit = matchup.move_limit + job.handicap = matchup.handicap + job.handicap_is_free = (matchup.handicap_style == 'free') + job.use_internal_scorer = (matchup.scorer == 'internal') + job.internal_scorer_handicap_compensation = \ + matchup.internal_scorer_handicap_compensation + job.sgf_event = matchup.event_description + return job + + def process_game_result(self, response): + self.engine_names.update(response.engine_names) + self.engine_descriptions.update(response.engine_descriptions) + matchup_id, game_number = response.game_data + game_id = response.game_id + self.working_matchups.add(matchup_id) + self.probationary_matchups.discard(matchup_id) + self.scheduler.fix(matchup_id, game_number) + self.results[matchup_id].append(response.game_result) + self.log_history("%7s %s" % (game_id, response.game_result.describe())) + + def process_game_error(self, job, previous_error_count): + # ignoring previous_error_count, as we can consider all jobs for the + # same matchup to be equivalent. + stop_competition = False + retry_game = False + matchup_id, game_data = job.game_data + if (matchup_id not in self.working_matchups or + matchup_id in self.probationary_matchups): + stop_competition = True + else: + self.probationary_matchups.add(matchup_id) + retry_game = True + return stop_competition, retry_game + + def write_matchup_report(self, out, matchup, results): + """Write the summary block for the specified matchup to 'out' + + results -- nonempty list of Game_results + + """ + # The control file might have changed since the results were recorded. + # We are guaranteed that the player codes correspond, but nothing else. + + # We use the current matchup to describe 'background' information, as + # that isn't available any other way, but we look to the results where + # we can. + + ms = tournament_results.Matchup_stats( + results, matchup.player_1, matchup.player_2) + ms.calculate_colour_breakdown() + ms.calculate_time_stats() + tournament_results.write_matchup_summary(out, matchup, ms) + + def write_matchup_reports(self, out): + """Write summary blocks for all live matchups to 'out'. + + This doesn't include ghost matchups, or matchups with no games. + + """ + first = True + for matchup in self.matchup_list: + results = self.results[matchup.id] + if not results: + continue + if first: + first = False + else: + print >>out + self.write_matchup_report(out, matchup, results) + + def write_ghost_matchup_reports(self, out): + """Write summary blocks for all ghost matchups to 'out'. + + (This may produce no output. Starts with a blank line otherwise.) + + """ + for matchup_id, matchup in sorted(self.ghost_matchups.iteritems()): + print >>out + results = self.results[matchup_id] + self.write_matchup_report(out, matchup, results) + + def write_player_descriptions(self, out): + """Write descriptions of all players to 'out'.""" + for code, description in sorted(self.engine_descriptions.items()): + print >>out, ("player %s: %s" % (code, description)) + + def get_tournament_results(self): + return tournament_results.Tournament_results( + self.matchup_list, self.results) + diff --git a/gomill/gomill/utils.py b/gomill/gomill/utils.py new file mode 100644 index 0000000..93598ad --- /dev/null +++ b/gomill/gomill/utils.py @@ -0,0 +1,74 @@ +"""Domain-independent utility functions for gomill. + +This module is designed to be used with 'from utils import *'. + +This is for generic utilities; see common for Go-specific utility functions. + +""" + +from __future__ import division + +__all__ = ["format_float", "format_percent", "sanitise_utf8", "isinf", "isnan"] + +def format_float(f): + """Format a Python float in a friendly way. + + This is intended for values like komi or win counts, which will be either + integers or half-integers. + + """ + if f == int(f): + return str(int(f)) + else: + return str(f) + +def format_percent(n, baseline): + """Format a ratio as a percentage (showing two decimal places). + + Returns a string. + + Accepts baseline zero and returns '??' or '--'. + + """ + if baseline == 0: + if n == 0: + return "--" + else: + return "??" + return "%.2f%%" % (100 * n/baseline) + + +def sanitise_utf8(s): + """Ensure an 8-bit string is utf-8. + + s -- 8-bit string (or None) + + Returns the sanitised string. If the string was already valid utf-8, returns + the same object. + + This replaces bad characters with ascii question marks (I don't want to use + a unicode replacement character, because if this function is doing anything + then it's likely that there's a non-unicode setup involved somewhere, so it + probably wouldn't be helpful). + + """ + if s is None: + return None + try: + s.decode("utf-8") + except UnicodeDecodeError: + return (s.decode("utf-8", 'replace') + .replace(u"\ufffd", u"?") + .encode("utf-8")) + else: + return s + +try: + from math import isinf, isnan +except ImportError: + # Python < 2.6 + def isinf(f): + return (f == float("1e500") or f == float("-1e500")) + def isnan(f): + return (f != f) + diff --git a/gomill/gomill_tests/__init__.py b/gomill/gomill_tests/__init__.py new file mode 100644 index 0000000..2d97a81 --- /dev/null +++ b/gomill/gomill_tests/__init__.py @@ -0,0 +1 @@ +# gomill_tests package diff --git a/gomill/gomill_tests/allplayall_tests.py b/gomill/gomill_tests/allplayall_tests.py new file mode 100644 index 0000000..6137fdc --- /dev/null +++ b/gomill/gomill_tests/allplayall_tests.py @@ -0,0 +1,289 @@ +"""Tests for allplayalls.py""" + +from __future__ import with_statement + +from textwrap import dedent +import cPickle as pickle + +from gomill import competitions +from gomill import allplayalls +from gomill.gtp_games import Game_result +from gomill.game_jobs import Game_job, Game_job_result +from gomill.competitions import ( + Player_config, NoGameAvailable, CompetitionError, ControlFileError) +from gomill.allplayalls import Competitor_config + +from gomill_tests import competition_test_support +from gomill_tests import gomill_test_support +from gomill_tests import test_framework +from gomill_tests.competition_test_support import ( + fake_response, check_screen_report) + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def check_short_report(tc, comp, + expected_grid, expected_matchups, expected_players, + competition_name="testcomp"): + """Check that an allplayall's short report is as expected.""" + expected = ("allplayall: %s\n\n%s\n%s\n%s\n" % + (competition_name, expected_grid, + expected_matchups, expected_players)) + tc.assertMultiLineEqual(competition_test_support.get_short_report(comp), + expected) + +class Allplayall_fixture(test_framework.Fixture): + """Fixture setting up a Allplayall. + + attributes: + comp -- Allplayall + + """ + def __init__(self, tc, config=None): + if config is None: + config = default_config() + self.tc = tc + self.comp = allplayalls.Allplayall('testcomp') + self.comp.initialise_from_control_file(config) + self.comp.set_clean_status() + + def check_screen_report(self, expected): + """Check that the screen report is as expected.""" + check_screen_report(self.tc, self.comp, expected) + + def check_short_report(self, *args, **kwargs): + """Check that the short report is as expected.""" + check_short_report(self.tc, self.comp, *args, **kwargs) + + +def default_config(): + return { + 'players' : { + 't1' : Player_config("test1"), + 't2' : Player_config("test2"), + 't3' : Player_config("test3"), + }, + 'board_size' : 13, + 'komi' : 7.5, + 'competitors' : [ + Competitor_config('t1'), + Competitor_config('t2'), + 't3', + ], + } + + + +def test_default_config(tc): + comp = allplayalls.Allplayall('test') + config = default_config() + comp.initialise_from_control_file(config) + comp.set_clean_status() + tr = comp.get_tournament_results() + tc.assertListEqual(tr.get_matchup_ids(), ['AvB', 'AvC', 'BvC']) + mBvC = tr.get_matchup('BvC') + tc.assertEqual(mBvC.player_1, 't2') + tc.assertEqual(mBvC.player_2, 't3') + tc.assertEqual(mBvC.board_size, 13) + tc.assertEqual(mBvC.komi, 7.5) + tc.assertEqual(mBvC.move_limit, 1000) + tc.assertEqual(mBvC.scorer, 'players') + tc.assertEqual(mBvC.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(mBvC.number_of_games, None) + tc.assertIs(mBvC.alternating, True) + tc.assertIs(mBvC.handicap, None) + tc.assertEqual(mBvC.handicap_style, 'fixed') + +def test_basic_config(tc): + comp = allplayalls.Allplayall('test') + config = default_config() + config['description'] = "default\nconfig" + config['board_size'] = 9 + config['komi'] = 0.5 + config['move_limit'] = 200 + config['scorer'] = 'internal' + config['internal_scorer_handicap_compensation'] = 'short' + config['rounds'] = 20 + comp.initialise_from_control_file(config) + tc.assertEqual(comp.description, "default\nconfig") + comp.set_clean_status() + mBvC = comp.get_tournament_results().get_matchup('BvC') + tc.assertEqual(mBvC.player_1, 't2') + tc.assertEqual(mBvC.player_2, 't3') + tc.assertEqual(mBvC.board_size, 9) + tc.assertEqual(mBvC.komi, 0.5) + tc.assertEqual(mBvC.move_limit, 200) + tc.assertEqual(mBvC.scorer, 'internal') + tc.assertEqual(mBvC.internal_scorer_handicap_compensation, 'short') + tc.assertEqual(mBvC.number_of_games, 20) + tc.assertIs(mBvC.alternating, True) + tc.assertIs(mBvC.handicap, None) + tc.assertEqual(mBvC.handicap_style, 'fixed') + +def test_unknown_player(tc): + comp = allplayalls.Allplayall('test') + config = default_config() + config['competitors'].append('nonex') + tc.assertRaisesRegexp( + ControlFileError, "competitor nonex: unknown player", + comp.initialise_from_control_file, config) + +def test_duplicate_player(tc): + comp = allplayalls.Allplayall('test') + config = default_config() + config['competitors'].append('t2') + tc.assertRaisesRegexp( + ControlFileError, "duplicate competitor: t2", + comp.initialise_from_control_file, config) + +def test_game_id_format(tc): + config = default_config() + config['rounds'] = 1000 + fx = Allplayall_fixture(tc, config) + tc.assertEqual(fx.comp.get_game().game_id, 'AvB_000') + +def test_get_player_checks(tc): + fx = Allplayall_fixture(tc) + checks = fx.comp.get_player_checks() + tc.assertEqual(len(checks), 3) + tc.assertEqual(checks[0].board_size, 13) + tc.assertEqual(checks[0].komi, 7.5) + tc.assertEqual(checks[0].player.code, "t1") + tc.assertEqual(checks[0].player.cmd_args, ['test1']) + tc.assertEqual(checks[1].player.code, "t2") + tc.assertEqual(checks[1].player.cmd_args, ['test2']) + tc.assertEqual(checks[2].player.code, "t3") + tc.assertEqual(checks[2].player.cmd_args, ['test3']) + +def test_play(tc): + fx = Allplayall_fixture(tc) + tc.assertIsNone(fx.comp.description) + + job1 = fx.comp.get_game() + tc.assertIsInstance(job1, Game_job) + tc.assertEqual(job1.game_id, 'AvB_0') + tc.assertEqual(job1.player_b.code, 't1') + tc.assertEqual(job1.player_w.code, 't2') + tc.assertEqual(job1.board_size, 13) + tc.assertEqual(job1.komi, 7.5) + tc.assertEqual(job1.move_limit, 1000) + tc.assertIs(job1.use_internal_scorer, False) + tc.assertEqual(job1.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(job1.game_data, ('AvB', 0)) + tc.assertIsNone(job1.sgf_filename) + tc.assertIsNone(job1.sgf_dirname) + tc.assertIsNone(job1.void_sgf_dirname) + tc.assertEqual(job1.sgf_event, 'testcomp') + tc.assertIsNone(job1.gtp_log_pathname) + + job2 = fx.comp.get_game() + tc.assertIsInstance(job2, Game_job) + tc.assertEqual(job2.game_id, 'AvC_0') + tc.assertEqual(job2.player_b.code, 't1') + tc.assertEqual(job2.player_w.code, 't3') + + response1 = fake_response(job1, 'b') + fx.comp.process_game_result(response1) + response2 = fake_response(job2, None) + fx.comp.process_game_result(response2) + + expected_grid = dedent("""\ + 2 games played + + A B C + A t1 1-0 0.5-0.5 + B t2 0-1 0-0 + C t3 0.5-0.5 0-0 + """) + expected_matchups = dedent("""\ + t1 v t2 (1 games) + board size: 13 komi: 7.5 + wins + t1 1 100.00% (black) + t2 0 0.00% (white) + + t1 v t3 (1 games) + board size: 13 komi: 7.5 + wins + t1 0.5 50.00% (black) + t3 0.5 50.00% (white) + """) + expected_players = dedent("""\ + player t1: t1 engine:v1.2.3 + player t2: t2 engine + testdescription + player t3: t3 engine + testdescription + """) + fx.check_screen_report(expected_grid) + fx.check_short_report(expected_grid, expected_matchups, expected_players) + + avb_results = fx.comp.get_tournament_results().get_matchup_results('AvB') + tc.assertEqual(avb_results, [response1.game_result]) + +def test_play_many(tc): + config = default_config() + config['rounds'] = 30 + fx = Allplayall_fixture(tc, config) + + jobs = [fx.comp.get_game() for _ in xrange(57)] + for i in xrange(57): + response = fake_response(jobs[i], 'b') + fx.comp.process_game_result(response) + + fx.check_screen_report(dedent("""\ + 57/90 games played + + A B C + A t1 10-9 10-9 + B t2 9-10 10-9 + C t3 9-10 9-10 + """)) + + tc.assertEqual( + len(fx.comp.get_tournament_results().get_matchup_results('AvB')), 19) + + comp2 = competition_test_support.check_round_trip(tc, fx.comp, config) + jobs2 = [comp2.get_game() for _ in range(4)] + tc.assertListEqual([job.game_id for job in jobs2], + ['AvB_19', 'AvC_19', 'BvC_19', 'AvB_20']) + tr = comp2.get_tournament_results() + tc.assertEqual(len(tr.get_matchup_results('AvB')), 19) + ms = tr.get_matchup_stats('AvB') + tc.assertEqual(ms.total, 19) + tc.assertEqual(ms.wins_1, 10) + tc.assertIs(ms.alternating, True) + +def test_competitor_change(tc): + fx = Allplayall_fixture(tc) + status = pickle.loads(pickle.dumps(fx.comp.get_status())) + + config2 = default_config() + del config2['competitors'][2] + comp2 = allplayalls.Allplayall('testcomp') + comp2.initialise_from_control_file(config2) + with tc.assertRaises(CompetitionError) as ar: + comp2.set_status(status) + tc.assertEqual( + str(ar.exception), + "competitor has been removed from control file") + + config3 = default_config() + config3['players']['t4'] = Player_config("test4") + config3['competitors'][2] = 't4' + comp3 = allplayalls.Allplayall('testcomp') + comp3.initialise_from_control_file(config3) + with tc.assertRaises(CompetitionError) as ar: + comp3.set_status(status) + tc.assertEqual( + str(ar.exception), + "competitors have changed in the control file") + + config4 = default_config() + config4['players']['t4'] = Player_config("test4") + config4['competitors'].append('t4') + comp4 = allplayalls.Allplayall('testcomp') + comp4.initialise_from_control_file(config4) + comp4.set_status(status) + diff --git a/gomill/gomill_tests/board_test_data.py b/gomill/gomill_tests/board_test_data.py new file mode 100644 index 0000000..c2de42f --- /dev/null +++ b/gomill/gomill_tests/board_test_data.py @@ -0,0 +1,395 @@ +play_tests = [ + +# code, list of moves to play, board representation, simple ko point, score + +('blank', [ +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 0), + +('twostone', [ +"B B2", "W C2", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . # o . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 0), + +('many-groups-1-capture', [ +"B C3", "W D3", "B C5", "B C4", "W D4", "B H1", "B B9", "B J6", +"B A7", "B B7", "W A3", "W J2", "W H2", "W G2", "W J3", +"B F7", +"W E6", "W G8", "W G6", "W F8", "W E7", "W F6", "W G7", "W E8", +], """\ +9 . # . . . . . . . +8 . . . . o o o . . +7 # # . . o . o . . +6 . . . . o o o . # +5 . . # . . . . . . +4 . . # o . . . . . +3 o . # o . . . . o +2 . . . . . . o o o +1 . . . . . . . # . + A B C D E F G H J +""", None, -8), + +('corner-bl', [ +"B A1", "W B1", "W A2", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 o . . . . . . . . +1 . o . . . . . . . + A B C D E F G H J +""", None, -81), + +('corner-all', [ +"B A1", "W B1", "W A2", +"B J1", "W H1", "W J2", +"B A9", "W B9", "W A8", +"B J9", "W H9", "W J8", +], """\ +9 . o . . . . . o . +8 o . . . . . . . o +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 o . . . . . . . o +1 . o . . . . . o . + A B C D E F G H J +""", None, -81), + +('multiple', [ +"W D4", "B D3", "W C5", "B C4", "W E5", "B E4", +"B B5", "B F5", "B C6", "B E6", "B D7", "W D6", +"B D5", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . # . . . . . +6 . . # . # . . . . +5 . # . # . # . . . +4 . . # . # . . . . +3 . . . # . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 81), + +('large', [ +"W D2", "W G2", "W E3", "W F3", "W F4", "W D5", "W E5", "W F5", "B E2", "B F2", +"B D3", "B G3", "B D4", "B G4", "B C5", "B G5", "B D6", "B E6", "B F6", "B E4", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . # # # . . . +5 . . # . . . # . . +4 . . . # # . # . . +3 . . . # . . # . . +2 . . . o # # o . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 16), + +('pre-recapture', [ +"W A1", "W B1", "W B2", "W C2", "W D3", "W E3", "W A4", "W B4", "W C4", "W E4", +"B A2", "B D2", "B E2", "B A3", "B B3", "B F3", "B D4", "B F4", "B E5", +"B C3", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . # . . . . +4 o o o # . # . . . +3 # # # . . # . . . +2 # o o # # . . . . +1 o o . . . . . . . + A B C D E F G H J +""", None, 6), + +('recapture', [ +"W A1", "W B1", "W B2", "W C2", "W D3", "W E3", "W A4", "W B4", "W C4", "W E4", +"B A2", "B D2", "B E2", "B A3", "B B3", "B F3", "B D4", "B F4", "B E5", +"B C3", "W D3", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . # . . . . +4 o o o # . # . . . +3 . . . o . # . . . +2 . o o # # . . . . +1 o o . . . . . . . + A B C D E F G H J +""", None, -6), + +('self-capture-1', [ +"B D4", "B C5", "B E5", "B D6", "W D5", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . # . . . . . +5 . . # . # . . . . +4 . . . # . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 81), + +('self-capture-2', [ +"B D4", "B E4", "B C5", "B F5", "B D6", "B E6", "W D5", "W E5", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . # # . . . . +5 . . # . . # . . . +4 . . . # # . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 81), + +('self-capture-3', [ +"B D4", "B E4", "B F4", "B C5", "B G5", "B D6", "B E6", "B F6", +"W D5", "W F5", "W E5", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . # # # . . . +5 . . # . . . # . . +4 . . . # # # . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 81), + +('self-capture-many', [ +"B E2", "B D3", "B F3", "B D4", "B F4", "B G4", "B C5", "B H5", "B C6", +"B F6", "B G6", "B D7", "B E7", +"W E3", "W E4", "W D5", "W F5", "W G5", "W D6", "W E6", "W E5", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . # # . . . . +6 . . # . . # # . . +5 . . # . . . . # . +4 . . . # . # # . . +3 . . . # . # . . . +2 . . . . # . . . . +1 . . . . . . . . . + A B C D E F G H J +""", None, 81), + +('ko-corner', [ +"B A1", "B B2", "B A3", +"W B1", "W A2", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 # . . . . . . . . +2 o # . . . . . . . +1 . o . . . . . . . + A B C D E F G H J +""", 'A1', -1), + +('notko-twocaptured', [ +"B B2", "B B3", "B A4", +"W B1", "W A2", "W A3", +"B A1", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 # . . . . . . . . +3 . # . . . . . . . +2 . # . . . . . . . +1 # o . . . . . . . + A B C D E F G H J +""", None, 5), + +('notko-tworecaptured', [ +"B A1", "B B3", "B A4", +"W B1", "W B2", "W A3", +"B A2", +], """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 # . . . . . . . . +3 . # . . . . . . . +2 # o . . . . . . . +1 # o . . . . . . . + A B C D E F G H J +""", None, 3), + +] + + +score_tests = [ + +# code, board representation, score + +('empty', """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", 0), + +('onestone', """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . # . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", 81), + +('easy', """\ +9 . . . # o . . . . +8 . . . # o . . . . +7 . . . # o . . . . +6 . . . # o . . . . +5 . . . # o . . . . +4 . . . # o . . . . +3 . . . # o . . . . +2 . . . # o . . . . +1 . . . # o . . . . + A B C D E F G H J +""", -9), + +('spoilt', """\ +9 . . . # o . . . . +8 . . . # o . . . . +7 . . . # o . . . . +6 . . . # o . . . . +5 . . . # o . . # . +4 . . . # o . . . . +3 . . . # o . . . . +2 . . . # o . . . . +1 . . . # o . . . . + A B C D E F G H J +""", 28), + +('busy', """\ +9 . . o . . o # . # +8 . o o o o o # . # +7 . o . o o # # # o +6 o . o # o # o o o +5 o o # # # # # o o +4 . o o # . o # o . +3 o # # # o o o o o +2 . o o # # # o o . +1 o . o # . # o o o + A B C D E F G H J +""", -26), + +] + + +setup_tests = [ + +# code, black points, white points, empty points, diagram, is_legal + +('blank', + [], + [], + [], +"""\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", True), + +('simple', + ['D4', 'D5'], + ['B1', 'J9'], + [], +"""\ +9 . . . . . . . . o +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . # . . . . . +4 . . . # . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . o . . . . . . . + A B C D E F G H J +""", True), + +('illegal', + ['A8', 'B9'], + ['A9'], + [], +"""\ +9 . # . . . . . . . +8 # . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J +""", False), + +] diff --git a/gomill/gomill_tests/board_tests.py b/gomill/gomill_tests/board_tests.py new file mode 100644 index 0000000..05b29c3 --- /dev/null +++ b/gomill/gomill_tests/board_tests.py @@ -0,0 +1,184 @@ +"""Tests for boards.py and ascii_boards.py + +We test these together because it's convenient for later boards tests to use +ascii_boards facilities. + +""" + +from __future__ import with_statement + +from gomill.common import format_vertex, move_from_vertex +from gomill import ascii_boards +from gomill import boards + +from gomill_tests import gomill_test_support +from gomill_tests import board_test_data + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + for t in board_test_data.play_tests: + suite.addTest(Play_test_TestCase(*t)) + for t in board_test_data.score_tests: + suite.addTest(Score_test_TestCase(*t)) + for t in board_test_data.setup_tests: + suite.addTest(Setup_test_TestCase(*t)) + +def test_attributes(tc): + b = boards.Board(5) + tc.assertEqual(b.side, 5) + tc.assertEqual( + b.board_points, + [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), + (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), + (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), + (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), + (4, 0), (4, 1), (4, 2), (4, 3), (4, 4)]) + +def test_basics(tc): + b = boards.Board(9) + + tc.assertTrue(b.is_empty()) + tc.assertItemsEqual(b.list_occupied_points(), []) + + tc.assertEqual(b.get(2, 3), None) + b.play(2, 3, 'b') + tc.assertEqual(b.get(2, 3), 'b') + tc.assertFalse(b.is_empty()) + b.play(3, 4, 'w') + + with tc.assertRaises(ValueError): + b.play(3, 4, 'w') + + tc.assertItemsEqual(b.list_occupied_points(), + [('b', (2, 3)), ('w', (3, 4))]) + + +_9x9_expected = """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . o . . . . +3 . . . # . . . . . +2 . . . . . . . . . +1 . . . . . . . . . + A B C D E F G H J\ +""" + +_13x13_expected = """\ +13 . . . . . . . . . . . . . +12 . . . . . . . . . . . . . +11 . . . . . . . . . . . . . +10 . . . . . . . . . . . . . + 9 . . . . . . . . . . . . . + 8 . . . . . . . . . . . . . + 7 . . . . . . . . . . . . . + 6 . . . . . . . . . . . . . + 5 . . . . . . . . . . . . . + 4 . . . . o . . . . . . . . + 3 . . . # . . . . . . . . . + 2 . . . . . . . . . . . . . + 1 . . . . . . . . . . . . . + A B C D E F G H J K L M N\ +""" + +def test_render_board_9x9(tc): + b = boards.Board(9) + b.play(2, 3, 'b') + b.play(3, 4, 'w') + tc.assertDiagramEqual(ascii_boards.render_board(b), _9x9_expected) + +def test_render_board_13x13(tc): + b = boards.Board(13) + b.play(2, 3, 'b') + b.play(3, 4, 'w') + tc.assertDiagramEqual(ascii_boards.render_board(b), _13x13_expected) + +def test_interpret_diagram(tc): + b1 = boards.Board(9) + b1.play(2, 3, 'b') + b1.play(3, 4, 'w') + b2 = ascii_boards.interpret_diagram(_9x9_expected, 9) + tc.assertEqual(b1, b2) + b3 = boards.Board(9) + b4 = ascii_boards.interpret_diagram(_9x9_expected, 9, b3) + tc.assertIs(b3, b4) + tc.assertEqual(b1, b3) + tc.assertRaisesRegexp(ValueError, "board not empty", + ascii_boards.interpret_diagram, _9x9_expected, 9, b3) + b5 = boards.Board(19) + tc.assertRaisesRegexp(ValueError, "wrong board size, must be 9$", + ascii_boards.interpret_diagram, _9x9_expected, 9, b5) + + tc.assertRaises(ValueError, ascii_boards.interpret_diagram, "nonsense", 9) + b6 = ascii_boards.interpret_diagram(_13x13_expected, 13) + tc.assertDiagramEqual(ascii_boards.render_board(b6), _13x13_expected) + +def test_copy(tc): + b1 = boards.Board(9) + b1.play(2, 3, 'b') + b1.play(3, 4, 'w') + b2 = b1.copy() + tc.assertEqual(b1, b2) + b2.play(5, 5, 'b') + b2.play(2, 1, 'b') + tc.assertNotEqual(b1, b2) + b1.play(5, 5, 'b') + b1.play(2, 1, 'b') + tc.assertEqual(b1, b2) + + +class Play_test_TestCase(gomill_test_support.Gomill_ParameterisedTestCase): + """Check final position reached by playing a sequence of moves.""" + test_name = "play_test" + parameter_names = ('moves', 'diagram', 'ko_vertex', 'score') + + def runTest(self): + b = boards.Board(9) + ko_point = None + for move in self.moves: + colour, vertex = move.split() + colour = colour.lower() + row, col = move_from_vertex(vertex, b.side) + ko_point = b.play(row, col, colour) + self.assertDiagramEqual(ascii_boards.render_board(b), + self.diagram.rstrip()) + if ko_point is None: + ko_vertex = None + else: + ko_vertex = format_vertex(ko_point) + self.assertEqual(ko_vertex, self.ko_vertex, "wrong ko point") + self.assertEqual(b.area_score(), self.score, "wrong score") + + +class Score_test_TestCase(gomill_test_support.Gomill_ParameterisedTestCase): + """Check score of a diagram.""" + test_name = "score_test" + parameter_names = ('diagram', 'score') + + def runTest(self): + b = ascii_boards.interpret_diagram(self.diagram, 9) + self.assertEqual(b.area_score(), self.score, "wrong score") + + +class Setup_test_TestCase(gomill_test_support.Gomill_ParameterisedTestCase): + """Check apply_setup().""" + test_name = "setup_test" + parameter_names = ('black_points', 'white_points', 'empty_points', + 'diagram', 'is_legal') + + def runTest(self): + def _interpret(moves): + return [move_from_vertex(v, b.side) for v in moves] + + b = boards.Board(9) + is_legal = b.apply_setup(_interpret(self.black_points), + _interpret(self.white_points), + _interpret(self.empty_points)) + self.assertDiagramEqual(ascii_boards.render_board(b), + self.diagram.rstrip()) + if self.is_legal: + self.assertTrue(is_legal, "setup should be considered legal") + else: + self.assertFalse(is_legal, "setup should be considered illegal") diff --git a/gomill/gomill_tests/cem_tuner_tests.py b/gomill/gomill_tests/cem_tuner_tests.py new file mode 100644 index 0000000..1833129 --- /dev/null +++ b/gomill/gomill_tests/cem_tuner_tests.py @@ -0,0 +1,227 @@ +"""Tests for cem_tuners.py""" + +from __future__ import with_statement, division + +import cPickle as pickle +from textwrap import dedent + +from gomill import cem_tuners +from gomill.game_jobs import Game_job, Game_job_result +from gomill.gtp_games import Game_result +from gomill.cem_tuners import Parameter_config +from gomill.competitions import ( + Player_config, CompetitionError, ControlFileError) + +from gomill_tests import gomill_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +def simple_make_candidate(*args): + if -1 in args: + raise ValueError("oops") + return Player_config("cand " + " ".join(map(str, args))) + +def clip_axisb(f): + f = float(f) + return max(0.0, min(100.0, f)) + +def default_config(): + return { + 'board_size' : 13, + 'komi' : 7.5, + 'players' : { + 'opp' : Player_config("test"), + }, + 'candidate_colour' : 'w', + 'opponent' : 'opp', + 'parameters' : [ + Parameter_config( + 'axisa', + initial_mean = 0.5, + initial_variance = 1.0, + format = "axa %.3f"), + Parameter_config( + 'axisb', + initial_mean = 50.0, + initial_variance = 1000.0, + transform = clip_axisb, + format = "axb %.1f"), + ], + 'batch_size' : 3, + 'samples_per_generation' : 4, + 'number_of_generations' : 3, + 'elite_proportion' : 0.1, + 'step_size' : 0.8, + 'make_candidate' : simple_make_candidate, + } + +def test_parameter_config(tc): + comp = cem_tuners.Cem_tuner('cemtest') + config = default_config() + comp.initialise_from_control_file(config) + + tc.assertEqual(comp.initial_distribution.format(), + " 0.50~1.00 50.00~1000.00") + tc.assertEqual(comp.format_engine_parameters((0.5, 23)), + "axa 0.500; axb 23.0") + tc.assertEqual(comp.format_engine_parameters(('x', 23)), + "[axisa?x]; axb 23.0") + tc.assertEqual(comp.format_optimiser_parameters((0.5, 500)), + "axa 0.500; axb 100.0") + + tc.assertEqual(comp.transform_parameters((0.5, 1e6)), (0.5, 100.0)) + with tc.assertRaises(CompetitionError) as ar: + comp.transform_parameters((0.5, None)) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + error from transform for axisb + TypeError: expected-float + traceback (most recent call last): + cem_tuner_tests|clip_axisb + failing line: + f = float(f) + """), fixups=[ + ("float() argument must be a string or a number", "expected-float"), + ("expected float, got NoneType object", "expected-float"), + ]) + + tc.assertRaisesRegexp( + ValueError, "'initial_variance': must be nonnegative", + comp.parameter_spec_from_config, + Parameter_config('pa1', initial_mean=0, + initial_variance=-1, format="%s")) + tc.assertRaisesRegexp( + ControlFileError, "'format': invalid format string", + comp.parameter_spec_from_config, + Parameter_config('pa1', initial_mean=0, initial_variance=1, + format="nopct")) + +def test_nonsense_parameter_config(tc): + comp = cem_tuners.Cem_tuner('cemtest') + config = default_config() + config['parameters'].append(99) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + 'parameters': item 2: not a Parameter""")) + +def test_transform_check(tc): + comp = cem_tuners.Cem_tuner('cemtest') + config = default_config() + config['parameters'][0] = Parameter_config( + 'axisa', + initial_mean = 0.5, + initial_variance = 1.0, + transform = str.split) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + parameter axisa: error from transform (applied to initial_mean) + TypeError: split-wants-float-not-str + traceback (most recent call last): + """), fixups=[ + ("descriptor 'split' requires a 'str' object but received a 'float'", + "split-wants-float-not-str"), + ("unbound method split() must be called with str instance as " + "first argument (got float instance instead)", + "split-wants-float-not-str"), + ]) + +def test_format_validation(tc): + comp = cem_tuners.Cem_tuner('cemtest') + config = default_config() + config['parameters'][0] = Parameter_config( + 'axisa', + initial_mean = 0.5, + initial_variance = 1.0, + transform = str, + format = "axa %f") + tc.assertRaisesRegexp( + ControlFileError, "'format': invalid format string", + comp.initialise_from_control_file, config) + +def test_make_candidate(tc): + comp = cem_tuners.Cem_tuner('cemtest') + config = default_config() + comp.initialise_from_control_file(config) + cand = comp.make_candidate('g0#1', (0.5, 23.0)) + tc.assertEqual(cand.code, 'g0#1') + tc.assertListEqual(cand.cmd_args, ['cand', '0.5', '23.0']) + with tc.assertRaises(CompetitionError) as ar: + comp.make_candidate('g0#1', (-1, 23)) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + error from make_candidate() + ValueError: oops + traceback (most recent call last): + cem_tuner_tests|simple_make_candidate + failing line: + raise ValueError("oops") + """)) + +def test_play(tc): + comp = cem_tuners.Cem_tuner('cemtest') + comp.initialise_from_control_file(default_config()) + comp.set_clean_status() + + tc.assertEqual(comp.generation, 0) + tc.assertEqual(comp.distribution.format(), + " 0.50~1.00 50.00~1000.00") + + job1 = comp.get_game() + tc.assertIsInstance(job1, Game_job) + tc.assertEqual(job1.game_id, 'g0#0r0') + tc.assertEqual(job1.player_b.code, 'g0#0') + tc.assertEqual(job1.player_w.code, 'opp') + tc.assertEqual(job1.board_size, 13) + tc.assertEqual(job1.komi, 7.5) + tc.assertEqual(job1.move_limit, 1000) + tc.assertIs(job1.use_internal_scorer, False) + tc.assertEqual(job1.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(job1.game_data, (0, 'g0#0', 0)) + tc.assertEqual(job1.sgf_event, 'cemtest') + tc.assertRegexpMatches(job1.sgf_note, '^Candidate parameters: axa ') + + job2 = comp.get_game() + tc.assertIsInstance(job2, Game_job) + tc.assertEqual(job2.game_id, 'g0#1r0') + tc.assertEqual(job2.player_b.code, 'g0#1') + tc.assertEqual(job2.player_w.code, 'opp') + + tc.assertEqual(comp.wins, [0, 0, 0, 0]) + + result1 = Game_result({'b' : 'g0#0', 'w' : 'opp'}, 'b') + result1.sgf_result = "B+8.5" + response1 = Game_job_result() + response1.game_id = job1.game_id + response1.game_result = result1 + response1.engine_names = { + 'opp' : 'opp engine:v1.2.3', + 'g0#0' : 'candidate engine', + } + response1.engine_descriptions = { + 'opp' : 'opp engine:v1.2.3', + 'g0#0' : 'candidate engine description', + } + response1.game_data = job1.game_data + comp.process_game_result(response1) + + tc.assertEqual(comp.wins, [1, 0, 0, 0]) + + comp2 = cem_tuners.Cem_tuner('cemtest') + comp2.initialise_from_control_file(default_config()) + status = pickle.loads(pickle.dumps(comp.get_status())) + comp2.set_status(status) + tc.assertEqual(comp2.wins, [1, 0, 0, 0]) + + result2 = Game_result({'b' : 'g0#1', 'w' : 'opp'}, None) + result2.set_jigo() + response2 = Game_job_result() + response2.game_id = job2.game_id + response2.game_result = result2 + response2.engine_names = response1.engine_names + response2.engine_descriptions = response1.engine_descriptions + response2.game_data = job2.game_data + comp.process_game_result(response2) + + tc.assertEqual(comp.wins, [1, 0.5, 0, 0]) + diff --git a/gomill/gomill_tests/common_tests.py b/gomill/gomill_tests/common_tests.py new file mode 100644 index 0000000..81cc45b --- /dev/null +++ b/gomill/gomill_tests/common_tests.py @@ -0,0 +1,72 @@ +"""Tests for common.py.""" + +import string + +from gomill_tests import gomill_test_support + +from gomill import common + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def test_opponent_of(tc): + oo = common.opponent_of + tc.assertEqual(oo('b'), 'w') + tc.assertEqual(oo('w'), 'b') + tc.assertRaises(ValueError, oo, 'x') + tc.assertRaises(ValueError, oo, None) + tc.assertRaises(ValueError, oo, 'B') + +def test_colour_name(tc): + cn = common.colour_name + tc.assertEqual(cn('b'), 'black') + tc.assertEqual(cn('w'), 'white') + tc.assertRaises(ValueError, cn, 'x') + tc.assertRaises(ValueError, cn, None) + tc.assertRaises(ValueError, cn, 'B') + +def test_column_letters(tc): + tc.assertEqual(common.column_letters, + "".join(c for c in string.uppercase if c != 'I')) + +def test_format_vertex(tc): + fv = common.format_vertex + tc.assertEqual(fv(None), "pass") + tc.assertEqual(fv((0, 0)), "A1") + tc.assertEqual(fv((8, 8)), "J9") + tc.assertEqual(fv((1, 5)), "F2") + tc.assertEqual(fv((24, 24)), "Z25") + tc.assertRaises(ValueError, fv, (-1, 2)) + tc.assertRaises(ValueError, fv, (2, -1)) + tc.assertRaises(ValueError, fv, (25, 1)) + tc.assertRaises(ValueError, fv, (1, 25)) + +def test_format_vertex_list(tc): + fvl = common.format_vertex_list + tc.assertEqual(fvl([]), "") + tc.assertEqual(fvl([(0, 0)]), "A1") + tc.assertEqual(fvl([(0, 0), (1, 5)]), "A1,F2") + tc.assertEqual(fvl([(0, 0), None, (1, 5)]), "A1,pass,F2") + +def test_move_from_vertex(tc): + cv = common.move_from_vertex + tc.assertEqual(cv("pass", 9), None) + tc.assertEqual(cv("pAss", 9), None) + tc.assertEqual(cv("A1", 9), (0, 0)) + tc.assertEqual(cv("a1", 9), (0, 0)) + tc.assertEqual(cv("A01", 9), (0, 0)) + tc.assertEqual(cv("J9", 9), (8, 8)) + tc.assertEqual(cv("M11", 19), (10, 11)) + tc.assertRaises(ValueError, cv, "M11", 9) + tc.assertRaises(ValueError, cv, "K9", 9) + tc.assertRaises(ValueError, cv, "J10", 9) + tc.assertRaises(ValueError, cv, "I5", 9) + tc.assertRaises(ValueError, cv, "", 9) + tc.assertRaises(ValueError, cv, "29", 9) + tc.assertRaises(ValueError, cv, "@9", 9) + tc.assertRaises(ValueError, cv, "A-3", 9) + tc.assertRaises(ValueError, cv, None, 9) + tc.assertRaises(ValueError, cv, "A1", 0) + tc.assertRaises(ValueError, cv, "A1", 30) + diff --git a/gomill/gomill_tests/competition_scheduler_tests.py b/gomill/gomill_tests/competition_scheduler_tests.py new file mode 100644 index 0000000..3fbde64 --- /dev/null +++ b/gomill/gomill_tests/competition_scheduler_tests.py @@ -0,0 +1,98 @@ +"""Tests for competition_schedulers.py""" + +import cPickle as pickle + +from gomill import competition_schedulers + +from gomill_tests import gomill_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def test_simple(tc): + sc = competition_schedulers.Simple_scheduler() + + def issue(n): + result = [sc.issue() for _ in xrange(n)] + sc._check_consistent() + return result + + sc._check_consistent() + tc.assertEqual(issue(4), [0, 1, 2, 3]) + sc.fix(2) + sc._check_consistent() + sc.fix(1) + sc._check_consistent() + tc.assertEqual(sc.issue(), 4) + tc.assertEqual(sc.fixed, 2) + tc.assertEqual(sc.issued, 5) + sc.rollback() + sc._check_consistent() + tc.assertEqual(sc.issued, 2) + tc.assertEqual(sc.fixed, 2) + + tc.assertListEqual(issue(2), [0, 3]) + + sc.rollback() + sc._check_consistent() + + tc.assertListEqual(issue(4), [0, 3, 4, 5]) + sc.fix(3) + sc._check_consistent() + sc.fix(5) + sc._check_consistent() + tc.assertEqual(sc.issue(), 6) + sc._check_consistent() + + sc = pickle.loads(pickle.dumps(sc)) + sc._check_consistent() + sc.rollback() + sc._check_consistent() + + tc.assertListEqual(issue(6), [0, 4, 6, 7, 8, 9]) + tc.assertEqual(sc.issued, 10) + tc.assertEqual(sc.fixed, 4) + + +def test_grouped(tc): + sc = competition_schedulers.Group_scheduler() + def issue(n): + return [sc.issue() for _ in xrange(n)] + + sc.set_groups([('m1', 4), ('m2', None)]) + + tc.assertTrue(sc.nothing_issued_yet()) + tc.assertFalse(sc.all_fixed()) + + tc.assertListEqual(issue(3), [ + ('m1', 0), + ('m2', 0), + ('m1', 1), + ]) + + tc.assertFalse(sc.nothing_issued_yet()) + + sc.fix('m1', 1) + sc.rollback() + issued = issue(14) + tc.assertListEqual(issued, [ + ('m2', 0), + ('m1', 0), + ('m2', 1), + ('m1', 2), + ('m2', 2), + ('m1', 3), + ('m2', 3), + ('m2', 4), + ('m2', 5), + ('m2', 6), + ('m2', 7), + ('m2', 8), + ('m2', 9), + ('m2', 10), + ]) + tc.assertFalse(sc.all_fixed()) + for token in issued: + sc.fix(*token) + tc.assertTrue(sc.all_fixed()) diff --git a/gomill/gomill_tests/competition_test_support.py b/gomill/gomill_tests/competition_test_support.py new file mode 100644 index 0000000..c1f99b2 --- /dev/null +++ b/gomill/gomill_tests/competition_test_support.py @@ -0,0 +1,80 @@ +"""Test support code for testing Competitions and Ringmasters.""" + +import cPickle as pickle +from cStringIO import StringIO + +from gomill import game_jobs +from gomill import gtp_games + +def fake_response(job, winner): + """Produce a response for the specified job. + + job -- Game_job + winner -- winning colour (None for a jigo, 'unknown' for unknown result) + + The winning margin (if not a jigo) is 1.5. + + """ + players = {'b' : job.player_b.code, 'w' : job.player_w.code} + if winner == 'unknown': + winner = None + is_unknown = True + else: + is_unknown = False + result = gtp_games.Game_result(players, winner) + result.game_id = job.game_id + if winner is None: + if is_unknown: + result.sgf_result = "Void" + result.detail = "fake unknown result" + else: + result.set_jigo() + else: + result.sgf_result += "1.5" + response = game_jobs.Game_job_result() + response.game_id = job.game_id + response.game_result = result + response.engine_names = { + job.player_b.code : '%s engine:v1.2.3' % job.player_b.code, + job.player_w.code : '%s engine' % job.player_w.code, + } + response.engine_descriptions = { + job.player_b.code : '%s engine:v1.2.3' % job.player_b.code, + job.player_w.code : '%s engine\ntestdescription' % job.player_w.code, + } + response.game_data = job.game_data + response.warnings = [] + response.log_entries = [] + return response + +def get_screen_report(comp): + """Retrieve a competition's screen report.""" + out = StringIO() + comp.write_screen_report(out) + return out.getvalue() + +def get_short_report(comp): + """Retrieve a competition's short report.""" + out = StringIO() + comp.write_short_report(out) + return out.getvalue() + +def check_screen_report(tc, comp, expected): + """Check that a competition's screen report is as expected.""" + tc.assertMultiLineEqual(get_screen_report(comp), expected) + +def check_round_trip(tc, comp, config): + """Check that a competition round-trips through saved state. + + Makes a new Competition, loads it from comp's saved state, and checks that + the resulting screen report is identical. + + Returns the new Competition. + + """ + comp2 = comp.__class__(comp.competition_code) + comp2.initialise_from_control_file(config) + status = pickle.loads(pickle.dumps(comp.get_status())) + comp2.set_status(status) + check_screen_report(tc, comp2, get_screen_report(comp)) + return comp2 diff --git a/gomill/gomill_tests/competition_tests.py b/gomill/gomill_tests/competition_tests.py new file mode 100644 index 0000000..93efd17 --- /dev/null +++ b/gomill/gomill_tests/competition_tests.py @@ -0,0 +1,165 @@ +"""Tests for competitions.py""" + +import os + +from gomill import competitions +from gomill.competitions import ControlFileError, Player_config + +from gomill_tests import gomill_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +def test_global_config(tc): + comp = competitions.Competition('test') + config = { + 'description' : "\nsome\ndescription ", + 'players' : {}, + } + comp.initialise_from_control_file(config) + tc.assertEqual(comp.description, "some\ndescription") + +def test_player_config(tc): + comp = competitions.Competition('test') + p1 = comp.game_jobs_player_from_config('pp', Player_config("cmd")) + tc.assertEqual(p1.code, 'pp') + tc.assertEqual(p1.cmd_args, ["cmd"]) + p2 = comp.game_jobs_player_from_config('pp', Player_config(command="cmd")) + tc.assertEqual(p2.code, 'pp') + tc.assertEqual(p2.cmd_args, ["cmd"]) + + tc.assertRaisesRegexp( + Exception, "'command' not specified", + comp.game_jobs_player_from_config, 'pp', + Player_config()) + tc.assertRaisesRegexp( + Exception, "too many positional arguments", + comp.game_jobs_player_from_config, 'pp', + Player_config("cmd", "xxx")) + tc.assertRaisesRegexp( + Exception, "command specified as both positional and keyword argument", + comp.game_jobs_player_from_config, 'pp', + Player_config("cmd", command="cmd2")) + tc.assertRaisesRegexp( + Exception, "unknown argument 'unexpected'", + comp.game_jobs_player_from_config, 'pp', + Player_config("cmd", unexpected=3)) + +def test_bad_player(tc): + comp = competitions.Competition('test') + config = { + 'players' : { + 't1' : Player_config("test"), + 't2' : None, + } + } + tc.assertRaisesRegexp( + ControlFileError, "'players': bad value for 't2': not a Player", + comp.initialise_from_control_file, config) + +def test_player_command(tc): + comp = competitions.Competition('test') + comp.set_base_directory("/base") + config = { + 'players' : { + 't1' : Player_config("test"), + 't2' : Player_config("/bin/test foo"), + 't3' : Player_config(["bin/test", "foo"]), + 't4' : Player_config("~/test foo"), + 't5' : Player_config("~root"), + } + } + comp.initialise_from_control_file(config) + tc.assertEqual(comp.players['t1'].cmd_args, ["test"]) + tc.assertEqual(comp.players['t2'].cmd_args, ["/bin/test", "foo"]) + tc.assertEqual(comp.players['t3'].cmd_args, ["/base/bin/test", "foo"]) + tc.assertEqual(comp.players['t4'].cmd_args, + [os.path.expanduser("~") + "/test", "foo"]) + tc.assertEqual(comp.players['t5'].cmd_args, ["~root"]) + +def test_player_is_reliable_scorer(tc): + comp = competitions.Competition('test') + config = { + 'players' : { + 't1' : Player_config("test"), + 't2' : Player_config("test", is_reliable_scorer=False), + 't3' : Player_config("test", is_reliable_scorer=True), + } + } + comp.initialise_from_control_file(config) + tc.assertTrue(comp.players['t1'].is_reliable_scorer) + tc.assertFalse(comp.players['t2'].is_reliable_scorer) + tc.assertTrue(comp.players['t3'].is_reliable_scorer) + +def test_player_cwd(tc): + comp = competitions.Competition('test') + comp.set_base_directory("/base") + config = { + 'players' : { + 't1' : Player_config("test"), + 't2' : Player_config("test", cwd="/abs"), + 't3' : Player_config("test", cwd="rel/sub"), + 't4' : Player_config("test", cwd="."), + 't5' : Player_config("test", cwd="~/tmp/sub"), + } + } + comp.initialise_from_control_file(config) + tc.assertIsNone(comp.players['t1'].cwd) + tc.assertEqual(comp.players['t2'].cwd, "/abs") + tc.assertEqual(comp.players['t3'].cwd, "/base/rel/sub") + tc.assertEqual(comp.players['t4'].cwd, "/base/.") + tc.assertEqual(comp.players['t5'].cwd, os.path.expanduser("~") + "/tmp/sub") + +def test_player_stderr(tc): + comp = competitions.Competition('test') + config = { + 'players' : { + 't1' : Player_config("test"), + 't2' : Player_config("test", discard_stderr=True), + 't3' : Player_config("test", discard_stderr=False), + } + } + comp.initialise_from_control_file(config) + tc.assertIs(comp.players['t1'].discard_stderr, False) + tc.assertEqual(comp.players['t2'].discard_stderr, True) + tc.assertIs(comp.players['t3'].discard_stderr, False) + +def test_player_startup_gtp_commands(tc): + comp = competitions.Competition('test') + config = { + 'players' : { + 't1' : Player_config( + "test", startup_gtp_commands=["foo"]), + 't2' : Player_config( + "test", startup_gtp_commands=["foo bar baz"]), + 't3' : Player_config( + "test", startup_gtp_commands=[["foo", "bar", "baz"]]), + 't4' : Player_config( + "test", startup_gtp_commands=[ + "xyzzy test", + ["foo", "bar", "baz"]]), + } + } + comp.initialise_from_control_file(config) + tc.assertListEqual(comp.players['t1'].startup_gtp_commands, + [("foo", [])]) + tc.assertListEqual(comp.players['t2'].startup_gtp_commands, + [("foo", ["bar", "baz"])]) + tc.assertListEqual(comp.players['t3'].startup_gtp_commands, + [("foo", ["bar", "baz"])]) + tc.assertListEqual(comp.players['t4'].startup_gtp_commands, + [("xyzzy", ["test"]), + ("foo", ["bar", "baz"])]) + +def test_player_gtp_aliases(tc): + comp = competitions.Competition('test') + config = { + 'players' : { + 't1' : Player_config( + "test", gtp_aliases={'foo' : 'bar', 'baz' : 'quux'}), + } + } + comp.initialise_from_control_file(config) + tc.assertDictEqual(comp.players['t1'].gtp_aliases, + {'foo' : 'bar', 'baz' : 'quux'}) + diff --git a/gomill/gomill_tests/game_job_tests.py b/gomill/gomill_tests/game_job_tests.py new file mode 100644 index 0000000..fd68545 --- /dev/null +++ b/gomill/gomill_tests/game_job_tests.py @@ -0,0 +1,458 @@ +"""Tests for game_jobs.py""" + +from __future__ import with_statement + +import os +from textwrap import dedent + +from gomill import gtp_controller +from gomill import game_jobs +from gomill.gtp_engine import GtpError, GtpFatalError +from gomill.job_manager import JobFailed + +from gomill_tests import test_framework +from gomill_tests import gomill_test_support +from gomill_tests import gtp_engine_fixtures + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +### Game_job proper + +class Test_game_job(game_jobs.Game_job): + """Variant of Game_job that doesn't write sgf files. + + Additional attributes: + _sgf_pathname_written -- pathname sgf file would have been written to + _sgf_written -- contents that would have been written + _mkdir_pathname -- directory pathname that would have been created + + """ + def __init__(self, *args, **kwargs): + game_jobs.Game_job.__init__(self, *args, **kwargs) + self._sgf_pathname_written = None + self._sgf_written = None + self._mkdir_pathname = None + + def _write_sgf(self, pathname, sgf_string): + self._sgf_pathname_written = pathname + self._sgf_written = sgf_string + + def _mkdir(self, pathname): + self._mkdir_pathname = pathname + + def _get_sgf_written(self): + """Return the 'scrubbed' sgf contents.""" + return gomill_test_support.scrub_sgf(self._sgf_written) + +class Game_job_fixture(test_framework.Fixture): + """Fixture setting up a Game_job. + + attributes: + job -- game_jobs.Game_job (in fact, a Test_game_job) + + """ + def __init__(self, tc): + player_b = game_jobs.Player() + player_b.code = 'one' + player_b.cmd_args = ['test', 'id=one'] + player_w = game_jobs.Player() + player_w.code = 'two' + player_w.cmd_args = ['test', 'id=two'] + self.job = Test_game_job() + self.job.game_id = 'gameid' + self.job.player_b = player_b + self.job.player_w = player_w + self.job.board_size = 9 + self.job.komi = 7.5 + self.job.move_limit = 1000 + self.job.sgf_dirname = "/sgf/test.games" + self.job.void_sgf_dirname = "/sgf/test.void" + self.job.sgf_filename = "gjtest.sgf" + +def test_player_copy(tc): + gj = Game_job_fixture(tc) + p1 = gj.job.player_b + p2 = p1.copy("clone") + tc.assertEqual(p2.code, "clone") + tc.assertEqual(p2.cmd_args, ['test', 'id=one']) + tc.assertIsNot(p1.cmd_args, p2.cmd_args) + +def test_game_job(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.game_data = 'gamedata' + gj.job.sgf_game_name = "gjt 0_000" + gj.job.sgf_event = "game_job_tests" + gj.job.sgf_note = "test sgf_note\non two lines" + result = gj.job.run() + # Win by 18 on the board minus 7.5 komi + tc.assertEqual(result.game_result.sgf_result, "B+10.5") + tc.assertEqual(result.game_id, 'gameid') + tc.assertEqual(result.game_result.game_id, 'gameid') + tc.assertEqual(result.game_data, 'gamedata') + tc.assertEqual(result.warnings, []) + tc.assertEqual(result.log_entries, []) + channel = fx.get_channel('one') + tc.assertIsNone(channel.requested_stderr) + tc.assertIsNone(channel.requested_cwd) + tc.assertIsNone(channel.requested_env) + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.games/gjtest.sgf') + tc.assertIsNone(gj.job._mkdir_pathname) + tc.assertMultiLineEqual(gj.job._get_sgf_written(), dedent("""\ + (;FF[4]AP[gomill:VER] + C[Event: game_job_tests + Game id gameid + Date *** + Result one beat two B+10.5 + test sgf_note + on two lines + Black one one + White two two] + CA[UTF-8]DT[***]EV[game_job_tests]GM[1]GN[gjt 0_000]KM[7.5]PB[one] + PW[two]RE[B+10.5]SZ[9];B[ei];W[gi];B[eh];W[gh];B[eg];W[gg];B[ef];W[gf];B[ee]; + W[ge];B[ed];W[gd];B[ec];W[gc];B[eb];W[gb];B[ea];W[ga];B[tt]; + C[one beat two B+10.5]W[tt]) + """)) + +def test_duplicate_player_codes(tc): + gj = Game_job_fixture(tc) + gj.job.player_w.code = "one" + tc.assertRaisesRegexp( + JobFailed, "error creating game: player codes must be distinct", + gj.job.run) + +def test_game_job_no_sgf(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.sgf_dirname = None + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "B+10.5") + tc.assertIsNone(gj.job._sgf_pathname_written) + +def test_game_job_forfeit(tc): + def handle_genmove(args): + raise GtpError("error") + def reject_genmove(channel): + channel.engine.add_command('genmove', handle_genmove) + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('reject_genmove', reject_genmove) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('init=reject_genmove') + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "B+F") + tc.assertEqual( + result.game_result.detail, + "forfeit: failure response from 'genmove w' to player two:\n" + "error") + tc.assertEqual( + result.warnings, + ["forfeit: failure response from 'genmove w' to player two:\n" + "error"]) + tc.assertEqual(result.log_entries, []) + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.games/gjtest.sgf') + +def test_game_job_forfeit_and_quit(tc): + def handle_genmove(args): + raise GtpFatalError("I'm out of here") + def reject_genmove(channel): + channel.engine.add_command('genmove', handle_genmove) + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('reject_genmove', reject_genmove) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('init=reject_genmove') + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "B+F") + tc.assertEqual( + result.game_result.detail, + "forfeit: failure response from 'genmove w' to player two:\n" + "I'm out of here") + tc.assertEqual( + result.warnings, + ["forfeit: failure response from 'genmove w' to player two:\n" + "I'm out of here"]) + tc.assertEqual( + result.log_entries, + ["error sending 'known_command gomill-cpu_time' to player two:\n" + "engine has closed the command channel"]) + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.games/gjtest.sgf') + +def test_game_job_exec_failure(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('fail=startup') + with tc.assertRaises(JobFailed) as ar: + gj.job.run() + tc.assertEqual(str(ar.exception), + "aborting game due to error:\n" + "error starting subprocess for player two:\n" + "exec forced to fail") + # No void sgf file unless at least one move was played + tc.assertIsNone(gj.job._sgf_pathname_written) + +def test_game_job_channel_error(tc): + def fail_first_genmove(channel): + channel.fail_command = 'genmove' + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('fail_first_genmove', fail_first_genmove) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('init=fail_first_genmove') + with tc.assertRaises(JobFailed) as ar: + gj.job.run() + tc.assertEqual(str(ar.exception), + "aborting game due to error:\n" + "transport error sending 'genmove w' to player two:\n" + "forced failure for send_command_line") + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.void/gjtest.sgf') + tc.assertEqual(gj.job._mkdir_pathname, '/sgf/test.void') + tc.assertMultiLineEqual(gj.job._get_sgf_written(), dedent("""\ + (;FF[4]AP[gomill:VER] + C[Game id gameid + Date *** + Black one one + White two two]CA[UTF-8] + DT[***]GM[1]GN[gameid]KM[7.5]PB[one]PW[two]RE[Void]SZ[9];B[ei] + C[aborting game due to error: + transport error sending 'genmove w' to player two: + forced failure for send_command_line] + ) + """)) + +def test_game_job_late_errors(tc): + def fail_close(channel): + channel.fail_close = True + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('fail_close', fail_close) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('init=fail_close') + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "B+10.5") + tc.assertEqual(result.warnings, []) + tc.assertEqual(result.log_entries, + ["error closing player two:\nforced failure for close"]) + +def test_game_job_late_error_from_void_game(tc): + def fail_genmove_and_close(channel): + channel.fail_command = 'genmove' + channel.fail_close = True + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('fail_genmove_and_close', fail_genmove_and_close) + gj = Game_job_fixture(tc) + gj.job.player_w.cmd_args.append('init=fail_genmove_and_close') + with tc.assertRaises(JobFailed) as ar: + gj.job.run() + tc.assertMultiLineEqual( + str(ar.exception), + "aborting game due to error:\n" + "transport error sending 'genmove w' to player two:\n" + "forced failure for send_command_line\n" + "also:\n" + "error closing player two:\n" + "forced failure for close") + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.void/gjtest.sgf') + +def test_game_job_cwd_env(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.player_b.cwd = "/nonexistent_directory" + gj.job.player_b.environ = {'GOMILL_TEST' : 'gomill'} + result = gj.job.run() + channel = fx.get_channel('one') + tc.assertIsNone(channel.requested_stderr) + tc.assertEqual(channel.requested_cwd, "/nonexistent_directory") + tc.assertEqual(channel.requested_env['GOMILL_TEST'], 'gomill') + # Check environment was merged, not replaced + tc.assertIn('PATH', channel.requested_env) + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.games/gjtest.sgf') + +def test_game_job_stderr_discarded(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.player_b.discard_stderr = True + result = gj.job.run() + channel = fx.get_channel('one') + tc.assertIsInstance(channel.requested_stderr, file) + tc.assertEqual(channel.requested_stderr.name, os.devnull) + +def test_game_job_stderr_set(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.stderr_pathname = "/dev/full" + result = gj.job.run() + channel = fx.get_channel('one') + tc.assertIsInstance(channel.requested_stderr, file) + tc.assertEqual(channel.requested_stderr.name, "/dev/full") + +def test_game_job_stderr_set_and_discarded(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.player_b.discard_stderr = True + result = gj.job.run() + channel = fx.get_channel('one') + tc.assertIsInstance(channel.requested_stderr, file) + tc.assertEqual(channel.requested_stderr.name, os.devnull) + +def test_game_job_gtp_aliases(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + gj = Game_job_fixture(tc) + gj.job.player_w.gtp_aliases = {'genmove': 'fail'} + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "B+F") + +def test_game_job_claim(tc): + def handle_genmove_ex(args): + tc.assertIn('claim', args) + return "claim" + def register_genmove_ex(channel): + channel.engine.add_command('gomill-genmove_ex', handle_genmove_ex) + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('genmove_ex', register_genmove_ex) + gj = Game_job_fixture(tc) + gj.job.player_b.cmd_args.append('init=genmove_ex') + gj.job.player_w.cmd_args.append('init=genmove_ex') + gj.job.player_w.allow_claim = True + result = gj.job.run() + tc.assertEqual(result.game_result.sgf_result, "W+") + tc.assertEqual(gj.job._sgf_pathname_written, '/sgf/test.games/gjtest.sgf') + +def test_game_job_handicap(tc): + def handle_fixed_handicap(args): + return "D4 K10 D10" + def register_fixed_handicap(channel): + channel.engine.add_command('fixed_handicap', handle_fixed_handicap) + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('handicap', register_fixed_handicap) + gj = Game_job_fixture(tc) + gj.job.player_b.cmd_args.append('init=handicap') + gj.job.player_w.cmd_args.append('init=handicap') + gj.job.board_size = 13 + gj.job.handicap = 3 + gj.job.handicap_is_free = False + gj.job.internal_scorer_handicap_compensation = 'full' + result = gj.job.run() + # area score 53, less 7.5 komi, less 3 handicap compensation + tc.assertEqual(result.game_result.sgf_result, "B+42.5") + + + +### check_player + +class Player_check_fixture(test_framework.Fixture): + """Fixture setting up a Player_check. + + attributes: + player -- game_jobs.Player + check -- game_jobs.Player_check + + """ + def __init__(self, tc): + self.player = game_jobs.Player() + self.player.code = 'test' + self.player.cmd_args = ['test', 'id=test'] + self.check = game_jobs.Player_check() + self.check.player = self.player + self.check.board_size = 9 + self.check.komi = 7.0 + +def test_check_player(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + tc.assertEqual(game_jobs.check_player(ck.check), []) + channel = fx.get_channel('test') + tc.assertIsNone(channel.requested_stderr) + tc.assertIsNone(channel.requested_cwd) + tc.assertIsNone(channel.requested_env) + +def test_check_player_discard_stderr(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + tc.assertEqual(game_jobs.check_player(ck.check, discard_stderr=True), []) + channel = fx.get_channel('test') + tc.assertIsInstance(channel.requested_stderr, file) + tc.assertEqual(channel.requested_stderr.name, os.devnull) + +def test_check_player_boardsize_fails(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + engine = gtp_engine_fixtures.get_test_engine() + fx.register_engine('no_boardsize', engine) + ck = Player_check_fixture(tc) + ck.player.cmd_args.append('engine=no_boardsize') + with tc.assertRaises(game_jobs.CheckFailed) as ar: + game_jobs.check_player(ck.check) + tc.assertEqual(str(ar.exception), + "failure response from 'boardsize 9' to test:\n" + "unknown command") + +def test_check_player_startup_gtp_commands(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + ck.player.startup_gtp_commands = [('list_commands', []), + ('nonexistent', ['command'])] + with tc.assertRaises(game_jobs.CheckFailed) as ar: + game_jobs.check_player(ck.check) + tc.assertEqual(str(ar.exception), + "failure response from 'nonexistent command' to test:\n" + "unknown command") + +def test_check_player_nonexistent_cwd(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + ck.player.cwd = "/nonexistent/directory" + with tc.assertRaises(game_jobs.CheckFailed) as ar: + game_jobs.check_player(ck.check) + tc.assertEqual(str(ar.exception), + "bad working directory: /nonexistent/directory") + +def test_check_player_cwd(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + ck.player.cwd = "/" + tc.assertEqual(game_jobs.check_player(ck.check), []) + channel = fx.get_channel('test') + tc.assertEqual(channel.requested_cwd, "/") + +def test_check_player_env(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + ck.player.environ = {'GOMILL_TEST' : 'gomill'} + tc.assertEqual(game_jobs.check_player(ck.check), []) + channel = fx.get_channel('test') + tc.assertEqual(channel.requested_env['GOMILL_TEST'], 'gomill') + # Check environment was merged, not replaced + tc.assertIn('PATH', channel.requested_env) + +def test_check_player_exec_failure(tc): + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + ck = Player_check_fixture(tc) + ck.player.cmd_args.append('fail=startup') + with tc.assertRaises(game_jobs.CheckFailed) as ar: + game_jobs.check_player(ck.check) + tc.assertEqual(str(ar.exception), + "error starting subprocess for test:\n" + "exec forced to fail") + +def test_check_player_channel_error(tc): + def fail_first_command(channel): + channel.fail_next_command = True + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('fail_first_command', fail_first_command) + ck = Player_check_fixture(tc) + ck.player.cmd_args.append('init=fail_first_command') + with tc.assertRaises(game_jobs.CheckFailed) as ar: + game_jobs.check_player(ck.check) + tc.assertEqual(str(ar.exception), + "transport error sending first command (protocol_version) " + "to test:\n" + "forced failure for send_command_line") + +def test_check_player_channel_error_on_close(tc): + def fail_close(channel): + channel.fail_close = True + fx = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + fx.register_init_callback('fail_close', fail_close) + ck = Player_check_fixture(tc) + ck.player.cmd_args.append('init=fail_close') + tc.assertEqual(game_jobs.check_player(ck.check), + ["error closing test:\nforced failure for close"]) + diff --git a/gomill/gomill_tests/gomill_test_support.py b/gomill/gomill_tests/gomill_test_support.py new file mode 100644 index 0000000..6c0de22 --- /dev/null +++ b/gomill/gomill_tests/gomill_test_support.py @@ -0,0 +1,161 @@ +"""Gomill-specific test support code.""" + +import re + +from gomill import __version__ +from gomill_tests.test_framework import unittest2 +from gomill_tests import test_framework + +from gomill.common import * +from gomill import ascii_boards +from gomill import boards + +# This makes TestResult ignore lines from this module in tracebacks +__unittest = True + +def compare_boards(b1, b2): + """Check whether two boards have the same position. + + returns a pair (position_is_the_same, message) + + """ + if b1.side != b2.side: + raise ValueError("size is different: %s, %s" % (b1.side, b2.side)) + differences = [] + for row, col in b1.board_points: + if b1.get(row, col) != b2.get(row, col): + differences.append((row, col)) + if not differences: + return True, None + msg = "boards differ at %s" % " ".join(map(format_vertex, differences)) + try: + msg += "\n%s\n%s" % ( + ascii_boards.render_board(b1), ascii_boards.render_board(b2)) + except Exception: + pass + return False, msg + +def compare_diagrams(d1, d2): + """Compare two ascii board diagrams. + + returns a pair (strings_are_equal, message) + + (assertMultiLineEqual tends to look nasty for these, so we just show them + both in full) + + """ + if d1 == d2: + return True, None + return False, "diagrams differ:\n%s\n\n%s" % (d1, d2) + +def scrub_sgf(s): + """Normalise sgf string for convenience of testing. + + Replaces dates with '***', and 'gomill:<__version__>' with 'gomill:VER'. + + Be careful: gomill version length can affect line wrapping. Either + serialise with wrap=None or remove newlines before comparing. + + """ + s = re.sub(r"(?m)(?<=^Date ).*$", "***", s) + s = re.sub(r"(?<=DT\[)[-0-9]+(?=\])", "***", s) + s = re.sub(r"gomill:" + re.escape(__version__), "gomill:VER", s) + return s + + +traceback_line_re = re.compile( + r" .*/([a-z0-9_]+)\.pyc?:[0-9]+ \(([a-z0-9_]+)\)") + +class Gomill_testcase_mixin(object): + """TestCase mixin adding support for gomill-specific types. + + This adds: + assertBoardEqual + assertEqual and assertNotEqual for Boards + + """ + def init_gomill_testcase_mixin(self): + self.addTypeEqualityFunc(boards.Board, self.assertBoardEqual) + + def _format_message(self, msg, standardMsg): + # This is the same as _formatMessage from python 2.7 unittest; copying + # it because it's not part of the public API. + if not self.longMessage: + return msg or standardMsg + if msg is None: + return standardMsg + try: + return '%s : %s' % (standardMsg, msg) + except UnicodeDecodeError: + return '%s : %s' % (unittest2.util.safe_repr(standardMsg), + unittest2.util.safe_repr(msg)) + + def assertBoardEqual(self, b1, b2, msg=None): + are_equal, desc = compare_boards(b1, b2) + if not are_equal: + self.fail(self._format_message(msg, desc+"\n")) + + def assertDiagramEqual(self, d1, d2, msg=None): + are_equal, desc = compare_diagrams(d1, d2) + if not are_equal: + self.fail(self._format_message(msg, desc+"\n")) + + def assertNotEqual(self, first, second, msg=None): + if isinstance(first, boards.Board) and isinstance(second, boards.Board): + are_equal, _ = compare_boards(first, second) + if not are_equal: + return + msg = self._format_message(msg, 'boards have the same position') + raise self.failureException(msg) + super(Gomill_testcase_mixin, self).assertNotEqual(first, second, msg) + + def assertTracebackStringEqual(self, seen, expected, fixups=()): + """Compare two strings which include tracebacks. + + This is for comparing strings containing tracebacks from + the compact_tracebacks module. + + Replaces the traceback lines describing source locations with + '|', for robustness. + + fixups -- list of pairs of strings + (additional substitutions to make in the 'seen' string) + + """ + lines = seen.split("\n") + new_lines = [] + for l in lines: + match = traceback_line_re.match(l) + if match: + l = "|".join(match.groups()) + for a, b in fixups: + l = l.replace(a, b) + new_lines.append(l) + self.assertMultiLineEqual("\n".join(new_lines), expected) + + +class Gomill_SimpleTestCase( + Gomill_testcase_mixin, test_framework.SimpleTestCase): + """SimpleTestCase with the Gomill mixin.""" + def __init__(self, *args, **kwargs): + test_framework.SimpleTestCase.__init__(self, *args, **kwargs) + self.init_gomill_testcase_mixin() + +class Gomill_ParameterisedTestCase( + Gomill_testcase_mixin, test_framework.ParameterisedTestCase): + """ParameterisedTestCase with the Gomill mixin.""" + def __init__(self, *args, **kwargs): + test_framework.ParameterisedTestCase.__init__(self, *args, **kwargs) + self.init_gomill_testcase_mixin() + + +def make_simple_tests(source, prefix="test_"): + """Make test cases from a module's test_xxx functions. + + See test_framework for details. + + The test functions can use the Gomill_testcase_mixin enhancements. + + """ + return test_framework.make_simple_tests( + source, prefix, testcase_class=Gomill_SimpleTestCase) diff --git a/gomill/gomill_tests/gtp_controller_test_support.py b/gomill/gomill_tests/gtp_controller_test_support.py new file mode 100644 index 0000000..bca3dc3 --- /dev/null +++ b/gomill/gomill_tests/gtp_controller_test_support.py @@ -0,0 +1,141 @@ +"""Support code for gtp_controller_tests. + +This is also used by gtp_engine_fixtures (and so by gtp proxy tests). + +""" + +from gomill import gtp_controller +from gomill.gtp_controller import ( + GtpChannelError, GtpProtocolError, GtpTransportError, GtpChannelClosed, + BadGtpResponse) + +from gomill_tests import test_support +from gomill_tests.test_framework import SupporterError + + +class Preprogrammed_gtp_channel(gtp_controller.Subprocess_gtp_channel): + """A Linebased_gtp_channel with hardwired response stream. + + Instantiate with a string containing the complete response stream. + + This sends the contents of the response stream, irrespective of what + commands are received. + + Pass hangs_before_eof True to simulate an engine that doesn't close its + response pipe when the preprogrammed response data runs out. + + The command stream is available from get_command_stream(). + + """ + def __init__(self, response, hangs_before_eof=False): + gtp_controller.Linebased_gtp_channel.__init__(self) + self.command_pipe = test_support.Mock_writing_pipe() + self.response_pipe = test_support.Mock_reading_pipe(response) + self.response_pipe.hangs_before_eof = hangs_before_eof + + def close(self): + self.command_pipe.close() + self.response_pipe.close() + + def get_command_stream(self): + """Return the complete contents of the command stream sent so far.""" + return self.command_pipe.getvalue() + + def break_command_stream(self): + """Break the simulated pipe for the command stream.""" + self.command_pipe.simulate_broken_pipe() + + def break_response_stream(self): + """Break the simulated pipe for the response stream.""" + self.response_pipe.simulate_broken_pipe() + + +class Testing_gtp_channel(gtp_controller.Linebased_gtp_channel): + """Linebased GTP channel that runs an internal Gtp_engine. + + Instantiate with a Gtp_engine_protocol object. + + This is used for testing how controllers handle GtpChannelError. + + Public attributes: + engine -- the engine it was instantiated with + is_closed -- bool (closed() has been called without a forced error) + + This raises an error if sent two commands without requesting a response in + between, or if asked for a response when no command was sent since the last + response. (GTP permits stacking up commands, but Gtp_controller should never + do it, so we want to report it). + + Unlike Internal_gtp_channel, this runs the command as the point when it is + sent. + + If you send a command after the engine has exited, this raises + GtpChannelClosed. Set the attribute engine_exit_breaks_commands to False to + disable this behaviour (it will ignore the command and respond with EOF + instead). + + You can force errors by setting the following attributes: + fail_next_command -- bool (send_command_line raises GtpTransportError) + fail_command -- string (like fail_next_command, if command line + starts with this string) + fail_next_response -- bool (get_response_line raises GtpTransportError) + force_next_response -- string (get_response_line uses this string) + fail_close -- bool (close raises GtpTransportError) + + """ + def __init__(self, engine): + gtp_controller.Linebased_gtp_channel.__init__(self) + self.engine = engine + self.stored_response = "" + self.session_is_ended = False + self.is_closed = False + self.engine_exit_breaks_commands = True + self.fail_next_command = False + self.fail_next_response = False + self.force_next_response = None + self.fail_close = False + self.fail_command = None + + def send_command_line(self, command): + if self.is_closed: + raise SupporterError("channel is closed") + if self.stored_response != "": + raise SupporterError("two commands in a row") + if self.session_is_ended: + if self.engine_exit_breaks_commands: + raise GtpChannelClosed("engine has closed the command channel") + return + if self.fail_next_command: + self.fail_next_command = False + raise GtpTransportError("forced failure for send_command_line") + if self.fail_command and command.startswith(self.fail_command): + self.fail_command = None + raise GtpTransportError("forced failure for send_command_line") + cmd_list = command.strip().split(" ") + is_error, response, end_session = \ + self.engine.run_command(cmd_list[0], cmd_list[1:]) + if end_session: + self.session_is_ended = True + self.stored_response = ("? " if is_error else "= ") + response + "\n\n" + + def get_response_line(self): + if self.is_closed: + raise SupporterError("channel is closed") + if self.stored_response == "": + if self.session_is_ended: + return "" + raise SupporterError("response request without command") + if self.fail_next_response: + self.fail_next_response = False + raise GtpTransportError("forced failure for get_response_line") + if self.force_next_response is not None: + self.stored_response = self.force_next_response + self.force_next_response = None + line, self.stored_response = self.stored_response.split("\n", 1) + return line + "\n" + + def close(self): + if self.fail_close: + raise GtpTransportError("forced failure for close") + self.is_closed = True + diff --git a/gomill/gomill_tests/gtp_controller_tests.py b/gomill/gomill_tests/gtp_controller_tests.py new file mode 100644 index 0000000..e200c87 --- /dev/null +++ b/gomill/gomill_tests/gtp_controller_tests.py @@ -0,0 +1,698 @@ +"""Tests for gtp_controller.py""" + +from __future__ import with_statement + +import os + +from gomill import gtp_controller +from gomill.gtp_controller import ( + GtpChannelError, GtpProtocolError, GtpTransportError, GtpChannelClosed, + BadGtpResponse, Gtp_controller) + +from gomill_tests import gomill_test_support +from gomill_tests import gtp_controller_test_support +from gomill_tests import gtp_engine_fixtures +from gomill_tests.test_framework import SupporterError +from gomill_tests.gtp_controller_test_support import Preprogrammed_gtp_channel + + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + + +### Channel-level + +def test_linebased_channel(tc): + channel = Preprogrammed_gtp_channel("=\n\n=\n\n") + tc.assertEqual(channel.get_command_stream(), "") + channel.send_command("play", ["b", "a3"]) + tc.assertEqual(channel.get_command_stream(), "play b a3\n") + tc.assertEqual(channel.get_response(), (False, "")) + channel.send_command("quit", []) + tc.assertEqual(channel.get_command_stream(), "play b a3\nquit\n") + tc.assertEqual(channel.get_response(), (False, "")) + tc.assertRaisesRegexp( + GtpChannelClosed, "engine has closed the response channel", + channel.get_response) + channel.close() + +def test_linebased_channel_responses(tc): + channel = Preprogrammed_gtp_channel( + "= 2\n\n" + # failure response + "? unknown command\n\n" + # final response with no newlines + "= ok") + channel.send_command("protocol_version", []) + tc.assertEqual(channel.get_response(), (False, "2")) + channel.send_command("xyzzy", ["1", "2"]) + tc.assertEqual(channel.get_response(), (True, "unknown command")) + channel.send_command("quit", ["1", "2"]) + tc.assertEqual(channel.get_response(), (False, "ok")) + +def test_linebased_channel_response_cleaning(tc): + channel = Preprogrammed_gtp_channel( + # empty response + "=\n\n" + # whitespace-only response + "= \n\n" + # ignores CRs (GTP spec) + "= 1abc\rde\r\n\r\n" + # ignores extra blank lines (GTP spec) + "= 2abcde\n\n\n\n" + # strips control characters (GTP spec) + "= 3a\x7fbc\x00d\x07e\n\x01\n" + # converts tabs to spaces (GTP spec) + "= 4abc\tde\n\n" + # strips leading whitespace (channel docs) + "= \t 5abcde\n\n" + # strips trailing whitepace (channel docs) + "= 6abcde \t \n\n" + # doesn't strip whitespace in the middle of a multiline response + "= 7aaa \n bbb\tccc\nddd \t \n\n" + # passes high characters through + "= 8ab\xc3\xa7de\n\n" + # all this at once, in a failure response + "? a\raa \r\n b\rbb\tcc\x01c\nddd \t \n\n" + ) + tc.assertEqual(channel.get_response(), (False, "")) + tc.assertEqual(channel.get_response(), (False, "")) + tc.assertEqual(channel.get_response(), (False, "1abcde")) + tc.assertEqual(channel.get_response(), (False, "2abcde")) + tc.assertEqual(channel.get_response(), (False, "3abcde")) + tc.assertEqual(channel.get_response(), (False, "4abc de")) + tc.assertEqual(channel.get_response(), (False, "5abcde")) + tc.assertEqual(channel.get_response(), (False, "6abcde")) + tc.assertEqual(channel.get_response(), (False, "7aaa \n bbb ccc\nddd")) + tc.assertEqual(channel.get_response(), (False, "8ab\xc3\xa7de")) + tc.assertEqual(channel.get_response(), (True, "aaa \n bbb ccc\nddd")) + +def test_linebased_channel_invalid_responses(tc): + channel = Preprogrammed_gtp_channel( + # good response first, to get past the "isn't speaking GTP" checking + "=\n\n" + "ERROR\n\n" + "# comments not allowed in responses\n\n" + ) + tc.assertEqual(channel.get_response(), (False, "")) + tc.assertRaisesRegexp( + GtpProtocolError, "^no success/failure indication from engine: " + "first line is `ERROR`$", + channel.get_response) + tc.assertRaisesRegexp( + GtpProtocolError, "^no success/failure indication from engine: " + "first line is `#", + channel.get_response) + +def test_linebased_channel_without_response(tc): + channel = Preprogrammed_gtp_channel("") + channel.send_command("protocol_version", []) + tc.assertRaisesRegexp( + GtpChannelClosed, "^engine has closed the response channel$", + channel.get_response) + channel.close() + +def test_linebased_channel_with_usage_message_response(tc): + channel = Preprogrammed_gtp_channel( + "Usage: randomprogram [options]\n\nOptions:\n" + "--help show this help message and exit\n") + channel.send_command("protocol_version", []) + tc.assertRaisesRegexp( + GtpProtocolError, "^engine isn't speaking GTP: first byte is 'U'$", + channel.get_response) + channel.close() + +def test_linebased_channel_with_interactive_response(tc): + channel = Preprogrammed_gtp_channel("prompt> \n", hangs_before_eof=True) + channel.send_command("protocol_version", []) + tc.assertRaisesRegexp( + GtpProtocolError, "^engine isn't speaking GTP", channel.get_response) + channel.close() + +def test_linebased_channel_hang(tc): + # Correct behaviour for a GTP controller here is to wait for a newline. + # (Would be nice to have a timeout.) + # This serves as a check that the hangs_before_eof modelling is working. + channel = Preprogrammed_gtp_channel("=prompt> ", hangs_before_eof=True) + channel.send_command("protocol_version", []) + tc.assertRaisesRegexp( + SupporterError, "this would hang", channel.get_response) + channel.close() + +def test_linebased_channel_with_gmp_response(tc): + channel = Preprogrammed_gtp_channel("\x01\xa1\xa0\x80", + hangs_before_eof=True) + channel.send_command("protocol_version", []) + tc.assertRaisesRegexp( + GtpProtocolError, "appears to be speaking GMP", channel.get_response) + channel.close() + +def test_linebased_channel_with_broken_command_pipe(tc): + channel = Preprogrammed_gtp_channel( + "Usage: randomprogram [options]\n\nOptions:\n" + "--help show this help message and exit\n") + channel.break_command_stream() + tc.assertRaisesRegexp( + GtpChannelClosed, "^engine has closed the command channel$", + channel.send_command, "protocol_version", []) + channel.close() + +def test_linebased_channel_with_broken_response_pipe(tc): + channel = Preprogrammed_gtp_channel("= 2\n\n? unreached\n\n") + channel.send_command("protocol_version", []) + tc.assertEqual(channel.get_response(), (False, "2")) + channel.break_response_stream() + channel.send_command("list_commands", []) + tc.assertRaisesRegexp( + GtpChannelClosed, "^engine has closed the response channel$", + channel.get_response) + channel.close() + +def test_channel_command_validation(tc): + channel = Preprogrammed_gtp_channel("\n\n") + # empty command + tc.assertRaises(ValueError, channel.send_command, "", []) + # space in command + tc.assertRaises(ValueError, channel.send_command, "play b a3", []) + # space after command + tc.assertRaises(ValueError, channel.send_command, "play ", ["b", "a3"]) + # control character in command + tc.assertRaises(ValueError, channel.send_command, "pla\x01y", ["b", "a3"]) + # unicode command + tc.assertRaises(ValueError, channel.send_command, u"protocol_version", []) + # space in argument + tc.assertRaises(ValueError, channel.send_command, "play", ["b a3"]) + # unicode argument + tc.assertRaises(ValueError, channel.send_command, "play ", [u"b", "a3"]) + # high characters + channel.send_command("pl\xc3\xa1y", ["b", "\xc3\xa13"]) + tc.assertEqual(channel.get_command_stream(), "pl\xc3\xa1y b \xc3\xa13\n") + + +### Validating Testing_gtp_channel + +def test_testing_gtp_channel(tc): + engine = gtp_engine_fixtures.get_test_engine() + channel = gtp_controller_test_support.Testing_gtp_channel(engine) + channel.send_command("play", ["b", "a3"]) + tc.assertEqual(channel.get_response(), (True, "unknown command")) + channel.send_command("test", []) + tc.assertEqual(channel.get_response(), (False, "test response")) + channel.send_command("multiline", []) + tc.assertEqual(channel.get_response(), + (False, "first line \n second line\nthird line")) + channel.send_command("quit", []) + tc.assertEqual(channel.get_response(), (False, "")) + tc.assertRaisesRegexp( + GtpChannelClosed, "engine has closed the command channel", + channel.send_command, "quit", []) + channel.close() + +def test_testing_gtp_channel_alt(tc): + engine = gtp_engine_fixtures.get_test_engine() + channel = gtp_controller_test_support.Testing_gtp_channel(engine) + channel.engine_exit_breaks_commands = False + channel.send_command("test", []) + tc.assertEqual(channel.get_response(), (False, "test response")) + channel.send_command("quit", []) + tc.assertEqual(channel.get_response(), (False, "")) + channel.send_command("test", []) + tc.assertRaisesRegexp( + GtpChannelClosed, "engine has closed the response channel", + channel.get_response) + channel.close() + +def test_testing_gtp_channel_fatal_errors(tc): + engine = gtp_engine_fixtures.get_test_engine() + channel = gtp_controller_test_support.Testing_gtp_channel(engine) + channel.send_command("fatal", []) + tc.assertEqual(channel.get_response(), (True, "fatal error")) + tc.assertRaisesRegexp( + GtpChannelClosed, "engine has closed the response channel", + channel.get_response) + channel.close() + +def test_testing_gtp_channel_sequencing(tc): + engine = gtp_engine_fixtures.get_test_engine() + channel = gtp_controller_test_support.Testing_gtp_channel(engine) + tc.assertRaisesRegexp( + SupporterError, "response request without command", + channel.get_response) + channel.send_command("test", []) + tc.assertRaisesRegexp( + SupporterError, "two commands in a row", + channel.send_command, "test", []) + +def test_testing_gtp_force_error(tc): + engine = gtp_engine_fixtures.get_test_engine() + channel = gtp_controller_test_support.Testing_gtp_channel(engine) + channel.fail_next_command = True + tc.assertRaisesRegexp( + GtpTransportError, "forced failure for send_command_line", + channel.send_command, "test", []) + channel.send_command("test", []) + channel.fail_next_response = True + tc.assertRaisesRegexp( + GtpTransportError, "forced failure for get_response_line", + channel.get_response) + channel.force_next_response = "# error\n\n" + tc.assertRaisesRegexp( + GtpProtocolError, + "no success/failure indication from engine: first line is `# error`", + channel.get_response) + channel.fail_close = True + tc.assertRaisesRegexp( + GtpTransportError, "forced failure for close", + channel.close) + + +### Controller-level + +def test_controller(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertEqual(controller.name, 'player test') + tc.assertIs(controller.channel, channel) + tc.assertFalse(controller.channel_is_bad) + + tc.assertEqual(controller.do_command("test", "ab", "cd"), "args: ab cd") + with tc.assertRaises(BadGtpResponse) as ar: + controller.do_command("error") + tc.assertEqual(ar.exception.gtp_error_message, "normal error") + tc.assertEqual(ar.exception.gtp_command, "error") + tc.assertSequenceEqual(ar.exception.gtp_arguments, []) + tc.assertEqual(str(ar.exception), + "failure response from 'error' to player test:\n" + "normal error") + with tc.assertRaises(BadGtpResponse) as ar: + controller.do_command("fatal") + tc.assertFalse(controller.channel_is_bad) + + with tc.assertRaises(GtpChannelClosed) as ar: + controller.do_command("test") + tc.assertEqual(str(ar.exception), + "error sending 'test' to player test:\n" + "engine has closed the command channel") + tc.assertTrue(controller.channel_is_bad) + controller.close() + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_alt_exit(tc): + channel = gtp_engine_fixtures.get_test_channel() + channel.engine_exit_breaks_commands = False + controller = Gtp_controller(channel, 'player test') + controller.do_command("quit") + tc.assertFalse(controller.channel_is_bad) + with tc.assertRaises(GtpChannelClosed) as ar: + controller.do_command("test") + tc.assertEqual(str(ar.exception), + "error reading response to 'test' from player test:\n" + "engine has closed the response channel") + tc.assertTrue(controller.channel_is_bad) + controller.close() + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_first_command_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + with tc.assertRaises(BadGtpResponse) as ar: + controller.do_command("error") + tc.assertEqual( + str(ar.exception), + "failure response from first command (error) to player test:\n" + "normal error") + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_command_transport_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertFalse(controller.channel_is_bad) + channel.fail_next_command = True + with tc.assertRaises(GtpTransportError) as ar: + controller.do_command("test") + tc.assertEqual( + str(ar.exception), + "transport error sending 'test' to player test:\n" + "forced failure for send_command_line") + tc.assertTrue(controller.channel_is_bad) + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_response_transport_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertFalse(controller.channel_is_bad) + channel.fail_next_response = True + with tc.assertRaises(GtpTransportError) as ar: + controller.do_command("test") + tc.assertEqual( + str(ar.exception), + "transport error reading response to first command (test) " + "from player test:\n" + "forced failure for get_response_line") + tc.assertTrue(controller.channel_is_bad) + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_response_protocol_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertFalse(controller.channel_is_bad) + channel.force_next_response = "# error\n\n" + with tc.assertRaises(GtpProtocolError) as ar: + controller.do_command("test") + tc.assertEqual( + str(ar.exception), + "GTP protocol error reading response to 'test' from player test:\n" + "no success/failure indication from engine: first line is `# error`") + tc.assertTrue(controller.channel_is_bad) + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_close(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertFalse(controller.channel_is_closed) + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertFalse(controller.channel_is_closed) + tc.assertFalse(controller.channel.is_closed) + controller.close() + tc.assertTrue(controller.channel_is_closed) + tc.assertTrue(controller.channel.is_closed) + tc.assertRaisesRegexp(StandardError, "^channel is closed$", + controller.do_command, "test") + tc.assertRaisesRegexp(StandardError, "^channel is closed$", + controller.close) + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_close_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + channel.fail_close = True + with tc.assertRaises(GtpTransportError) as ar: + controller.close() + tc.assertEqual( + str(ar.exception), + "error closing player test:\n" + "forced failure for close") + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_safe_close(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertFalse(controller.channel_is_closed) + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertFalse(controller.channel_is_closed) + tc.assertFalse(controller.channel.is_closed) + controller.safe_close() + tc.assertTrue(controller.channel_is_closed) + tc.assertTrue(controller.channel.is_closed) + tc.assertListEqual(channel.engine.commands_handled, + [('test', []), ('quit', [])]) + # safe to call twice + controller.safe_close() + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_safe_close_after_error(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertFalse(controller.channel_is_bad) + channel.force_next_response = "# error\n\n" + with tc.assertRaises(GtpProtocolError) as ar: + controller.do_command("test") + tc.assertTrue(controller.channel_is_bad) + # doesn't send quit when channel_is_bad + controller.safe_close() + tc.assertTrue(controller.channel_is_closed) + tc.assertTrue(controller.channel.is_closed) + tc.assertListEqual(channel.engine.commands_handled, + [('test', []), ('test', [])]) + tc.assertListEqual(controller.retrieve_error_messages(), []) + +def test_controller_safe_close_with_error_from_quit(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + channel.force_next_response = "# error\n\n" + controller.safe_close() + tc.assertTrue(controller.channel_is_closed) + tc.assertTrue(controller.channel.is_closed) + tc.assertListEqual(channel.engine.commands_handled, + [('quit', [])]) + tc.assertListEqual( + controller.retrieve_error_messages(), + ["GTP protocol error reading response to first command (quit) " + "from player test:\n" + "no success/failure indication from engine: first line is `# error`"]) + +def test_controller_safe_close_with_failure_response_from_quit(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + channel.engine.force_error("quit") + controller.safe_close() + tc.assertTrue(controller.channel_is_closed) + tc.assertTrue(controller.channel.is_closed) + tc.assertListEqual(channel.engine.commands_handled, + [('quit', [])]) + error_messages = controller.retrieve_error_messages() + tc.assertEqual(len(error_messages), 1) + tc.assertEqual( + error_messages[0], + "failure response from first command (quit) to player test:\n" + "handler forced to fail") + +def test_controller_safe_close_with_error_from_close(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + channel.fail_close = True + controller.safe_close() + tc.assertTrue(controller.channel_is_closed) + tc.assertListEqual(channel.engine.commands_handled, + [('quit', [])]) + tc.assertListEqual( + controller.retrieve_error_messages(), + ["error closing player test:\n" + "forced failure for close"]) + +def test_safe_do_command(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + tc.assertEqual(controller.safe_do_command("test", "ab"), "args: ab") + with tc.assertRaises(BadGtpResponse) as ar: + controller.safe_do_command("error") + tc.assertFalse(controller.channel_is_bad) + channel.fail_next_response = True + tc.assertIsNone(controller.safe_do_command("test")) + tc.assertTrue(controller.channel_is_bad) + tc.assertIsNone(controller.safe_do_command("test")) + tc.assertListEqual( + controller.retrieve_error_messages(), + ["transport error reading response to 'test' from player test:\n" + "forced failure for get_response_line"]) + controller.safe_close() + # check that third 'test' wasn't sent, and nor was 'quit' + tc.assertListEqual(channel.engine.commands_handled, + [('test', ['ab']), ('error', []), ('test', [])]) + +def test_safe_do_command_closed_channel(tc): + # check it's ok to call safe_do_command() on a closed channel + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + controller.safe_close() + tc.assertIsNone(controller.safe_do_command("test")) + tc.assertListEqual(channel.engine.commands_handled, + [('quit', [])]) + tc.assertListEqual(controller.retrieve_error_messages(), []) + + +def test_known_command(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'kc test') + tc.assertTrue(controller.known_command("test")) + tc.assertFalse(controller.known_command("nonesuch")) + tc.assertTrue(controller.known_command("test")) + tc.assertFalse(controller.known_command("nonesuch")) + +def test_known_command_2(tc): + # Checking that known_command caches its responses + # and that it treats an error or unknown value the same as 'false'. + channel = Preprogrammed_gtp_channel( + "= true\n\n= absolutely not\n\n? error\n\n# unreached\n\n") + controller = Gtp_controller(channel, 'kc2 test') + tc.assertTrue(controller.known_command("one")) + tc.assertFalse(controller.known_command("two")) + tc.assertFalse(controller.known_command("three")) + tc.assertTrue(controller.known_command("one")) + tc.assertFalse(controller.known_command("two")) + tc.assertEqual( + channel.get_command_stream(), + "known_command one\nknown_command two\nknown_command three\n") + +def test_check_protocol_version(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'pv test') + controller.check_protocol_version() + +def test_check_protocol_version_2(tc): + channel = Preprogrammed_gtp_channel("= 1\n\n? error\n\n# unreached\n\n") + controller = Gtp_controller(channel, 'pv2 test') + with tc.assertRaises(BadGtpResponse) as ar: + controller.check_protocol_version() + tc.assertEqual(str(ar.exception), "pv2 test reports GTP protocol version 1") + tc.assertEqual(ar.exception.gtp_error_message, None) + # check error is not treated as a check failure + controller.check_protocol_version() + +def test_list_commands(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'lc test') + channel.engine.add_command("xyzzy", None) + channel.engine.add_command("pl ugh", None) + tc.assertListEqual( + controller.list_commands(), + ['error', 'fatal', 'known_command', 'list_commands', + 'multiline', 'protocol_version', 'quit', 'test', 'xyzzy']) + +def test_gtp_aliases(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'alias test') + controller.set_gtp_aliases({ + 'aliased' : 'test', + 'aliased2' : 'nonesuch', + }) + tc.assertIs(controller.known_command("test"), True) + tc.assertIs(controller.known_command("aliased"), True) + tc.assertIs(controller.known_command("nonesuch"), False) + tc.assertIs(controller.known_command("test"), True) + tc.assertIs(controller.known_command("aliased"), True) + tc.assertIs(controller.known_command("nonesuch"), False) + tc.assertEqual(controller.do_command("test"), "test response") + tc.assertEqual(controller.do_command("aliased"), "test response") + with tc.assertRaises(BadGtpResponse) as ar: + controller.do_command("aliased2") + tc.assertEqual(ar.exception.gtp_error_message, "unknown command") + tc.assertEqual(ar.exception.gtp_command, "nonesuch") + +def test_gtp_aliases_safe(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'alias test') + controller.set_gtp_aliases({ + 'aliased' : 'test', + 'aliased2' : 'nonesuch', + }) + tc.assertIs(controller.safe_known_command("test"), True) + tc.assertIs(controller.safe_known_command("aliased"), True) + tc.assertIs(controller.safe_known_command("nonesuch"), False) + tc.assertIs(controller.safe_known_command("test"), True) + tc.assertIs(controller.safe_known_command("aliased"), True) + tc.assertIs(controller.safe_known_command("nonesuch"), False) + tc.assertEqual(controller.safe_do_command("test"), "test response") + tc.assertEqual(controller.safe_do_command("aliased"), "test response") + with tc.assertRaises(BadGtpResponse) as ar: + controller.safe_do_command("aliased2") + tc.assertEqual(ar.exception.gtp_error_message, "unknown command") + tc.assertEqual(ar.exception.gtp_command, "nonesuch") + + +def test_fix_version(tc): + fv = gtp_controller._fix_version + tc.assertEqual(fv("foo", "bar"), "bar") + tc.assertEqual(fv("foo", "FOO bar"), "bar") + tc.assertEqual(fv("foo", "asd " * 16), "asd " * 16) + tc.assertEqual(fv("foo", "asd " * 17), "asd") + tc.assertEqual( + fv("MoGo", "MoGo release 1. Please read http://www.lri.fr/~gelly/MoGo.htm for more information. That is NOT an official developpement MoGo version, but it is a public release. Its strength highly depends on your hardware and the time settings."), + "release 1") + tc.assertEqual( + fv("Pachi UCT Engine", "8.99 (Hakugen-devel): I'm playing UCT. When I'm losing, I will resign, if I think I win, I play until you pass. Anyone can send me 'winrate' in private chat to get my assessment of the position."), + "8.99 (Hakugen-devel)") + +def test_describe_engine(tc): + channel = gtp_engine_fixtures.get_test_channel() + controller = Gtp_controller(channel, 'player test') + short_s, long_s = gtp_controller.describe_engine(controller) + tc.assertEqual(short_s, "unknown") + tc.assertEqual(long_s, "unknown") + + channel = gtp_engine_fixtures.get_test_channel() + channel.engine.add_command('name', lambda args:"test engine") + controller = Gtp_controller(channel, 'player test') + short_s, long_s = gtp_controller.describe_engine(controller) + tc.assertEqual(short_s, "test engine") + tc.assertEqual(long_s, "test engine") + + channel = gtp_engine_fixtures.get_test_channel() + channel.engine.add_command('name', lambda args:"test engine") + channel.engine.add_command('version', lambda args:"1.2.3") + controller = Gtp_controller(channel, 'player test') + short_s, long_s = gtp_controller.describe_engine(controller) + tc.assertEqual(short_s, "test engine:1.2.3") + tc.assertEqual(long_s, "test engine:1.2.3") + + channel = gtp_engine_fixtures.get_test_channel() + channel.engine.add_command('name', lambda args:"test engine") + channel.engine.add_command('version', lambda args:"1.2.3") + channel.engine.add_command( + 'gomill-describe_engine', + lambda args:"test engine (v1.2.3):\n pl\xc3\xa1yer \xa3") + controller = Gtp_controller(channel, 'player test') + short_s, long_s = gtp_controller.describe_engine(controller) + tc.assertEqual(short_s, "test engine:1.2.3") + tc.assertEqual(long_s, "test engine (v1.2.3):\n pl\xc3\xa1yer ?") + + channel = gtp_engine_fixtures.get_test_channel() + channel.engine.add_command('name', lambda args:"test engine") + channel.engine.add_command('version', lambda args:"test engine v1.2.3") + controller = Gtp_controller(channel, 'player test') + short_s, long_s = gtp_controller.describe_engine(controller) + tc.assertEqual(short_s, "test engine:v1.2.3") + tc.assertEqual(long_s, "test engine:v1.2.3") + + +### Subprocess-specific + +def test_subprocess_channel(tc): + # This tests that Subprocess_gtp_channel really launches a subprocess. + # It also checks that the 'stderr', 'env' and 'cwd' parameters work. + # This test relies on there being a 'python' executable on the PATH + # (doesn't have to be the same version as is running the testsuite). + fx = gtp_engine_fixtures.State_reporter_fixture(tc) + rd, wr = os.pipe() + try: + channel = gtp_controller.Subprocess_gtp_channel( + fx.cmd, + stderr=wr, + env={'GOMILL_TEST' : "from_gtp_controller_tests"}, + cwd="/") + tc.assertEqual(os.read(rd, 256), "subprocess_state_reporter: testing\n") + finally: + os.close(wr) + os.close(rd) + tc.assertIsNone(channel.exit_status) + tc.assertIsNone(channel.resource_usage) + channel.send_command("tell", []) + tc.assertEqual(channel.get_response(), + (False, "cwd: /\nGOMILL_TEST:from_gtp_controller_tests")) + channel.close() + tc.assertEqual(channel.exit_status, 0) + rusage = channel.resource_usage + tc.assertTrue(hasattr(rusage, 'ru_utime')) + tc.assertTrue(hasattr(rusage, 'ru_stime')) + +def test_subprocess_channel_nonexistent_program(tc): + with tc.assertRaises(GtpChannelError) as ar: + gtp_controller.Subprocess_gtp_channel(["/nonexistent/program"]) + tc.assertIn("[Errno 2] No such file or directory", str(ar.exception)) + +def test_subprocess_channel_with_controller(tc): + # Also tests that leaving 'env' and 'cwd' unset works + fx = gtp_engine_fixtures.State_reporter_fixture(tc) + channel = gtp_controller.Subprocess_gtp_channel(fx.cmd, stderr=fx.devnull) + controller = Gtp_controller(channel, 'subprocess test') + tc.assertEqual(controller.do_command("tell"), + "cwd: %s\nGOMILL_TEST:None" % os.getcwd()) + controller.close() + tc.assertEqual(channel.exit_status, 0) + rusage = channel.resource_usage + tc.assertTrue(hasattr(rusage, 'ru_utime')) + diff --git a/gomill/gomill_tests/gtp_engine_fixtures.py b/gomill/gomill_tests/gtp_engine_fixtures.py new file mode 100644 index 0000000..716ef1a --- /dev/null +++ b/gomill/gomill_tests/gtp_engine_fixtures.py @@ -0,0 +1,390 @@ +"""Engines (and channels) provided for the use of controller-side testing.""" + +import os + +from gomill import gtp_controller +from gomill import gtp_engine +from gomill.gtp_engine import GtpError, GtpFatalError +from gomill.gtp_controller import GtpChannelError +from gomill.common import * + +from gomill_tests import test_framework +from gomill_tests import gtp_controller_test_support +from gomill_tests.test_framework import SupporterError + + +## Test engine + +class Test_gtp_engine_protocol(gtp_engine.Gtp_engine_protocol): + """Variant of Gtp_engine_protocol with additional facilities for testing. + + Public attributes: + commands_handled -- list of pairs (command, args) + + This records all commands sent to the engine and makes them available in the + commands_handled attribute. It also provides a mechanism to force commands + to fail. + + """ + def __init__(self): + gtp_engine.Gtp_engine_protocol.__init__(self) + self.commands_handled = [] + + def run_command(self, command, args): + self.commands_handled.append((command, args)) + return gtp_engine.Gtp_engine_protocol.run_command(self, command, args) + + def _forced_error(self, args): + raise GtpError("handler forced to fail") + + def _forced_fatal_error(self, args): + raise GtpFatalError("handler forced to fail and exit") + + def force_error(self, command): + """Set the handler for 'command' to report failure.""" + self.add_command(command, self._forced_error) + + def force_fatal_error(self, command): + """Set the handler for 'command' to report failure and exit.""" + self.add_command(command, self._forced_fatal_error) + + +def get_test_engine(): + """Return a Gtp_engine_protocol useful for testing controllers. + + Actually returns a Test_gtp_engine_protocol. + + """ + + def handle_test(args): + if args: + return "args: " + " ".join(args) + else: + return "test response" + + def handle_multiline(args): + return "first line \n second line\nthird line" + + def handle_error(args): + raise GtpError("normal error") + + def handle_fatal_error(args): + raise GtpFatalError("fatal error") + + engine = Test_gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_command('test', handle_test) + engine.add_command('multiline', handle_multiline) + engine.add_command('error', handle_error) + engine.add_command('fatal', handle_fatal_error) + return engine + +def get_test_channel(): + """Return a Testing_gtp_channel connected to the test engine.""" + engine = get_test_engine() + return gtp_controller_test_support.Testing_gtp_channel(engine) + + +## Test player engine + +class Test_player(object): + """Trivial player. + + This supports at least the minimal commands required to play a game. + + At present, this plays up column 4 (for black) or 6 (for white), then + passes. It pays no attention to its opponent's moves, and doesn't maintain a + board position. + + (This means that if two Test_players play each other, black will win by + boardsize*2 on the board). + + """ + def __init__(self): + self.boardsize = None + self.row_to_play = 0 + + def handle_boardsize(self, args): + self.boardsize = gtp_engine.interpret_int(args[0]) + + def handle_clear_board(self, args): + pass + + def handle_komi(self, args): + pass + + def handle_play(self, args): + pass + + def handle_genmove(self, args): + colour = gtp_engine.interpret_colour(args[0]) + if self.row_to_play < self.boardsize: + col = 4 if colour == 'b' else 6 + result = format_vertex((self.row_to_play, col)) + self.row_to_play += 1 + return result + else: + return "pass" + + def handle_fail(self, args): + raise GtpError("test player forced to fail") + + def get_handlers(self): + return { + 'boardsize' : self.handle_boardsize, + 'clear_board' : self.handle_clear_board, + 'komi' : self.handle_komi, + 'play' : self.handle_play, + 'genmove' : self.handle_genmove, + 'fail' : self.handle_fail, + } + +class Programmed_player(object): + """Player that follows a preset sequence of moves. + + Instantiate with + moves -- a sequence of pairs (colour, vertex) + + The sequence can have moves for both colours; genmove goes through the + moves in order and ignores ones for the colour that wasn't requested (the + idea is that you can create two players with the same move list). + + Passes when it runs out of moves. + + if 'vertex' is a tuple, it's interpreted as (row, col) and converted to a + gtp vertex. The special value 'fail' causes a GtpError. Otherwise it's + returned literally. + + """ + def __init__(self, moves, reject=None): + self.moves = [] + for colour, vertex in moves: + if isinstance(vertex, tuple): + vertex = format_vertex(vertex) + self.moves.append((colour, vertex)) + self.reject = reject + self._reset() + + def _reset(self): + self.iter = iter(self.moves) + + def handle_boardsize(self, args): + pass + + def handle_clear_board(self, args): + self._reset() + + def handle_komi(self, args): + pass + + def handle_play(self, args): + if self.reject is None: + return + vertex, msg = self.reject + if args[1].lower() == vertex.lower(): + raise GtpError(msg) + + def handle_genmove(self, args): + colour = gtp_engine.interpret_colour(args[0]) + for move_colour, vertex in self.iter: + if move_colour == colour: + if vertex == 'fail': + raise GtpError("forced to fail") + return vertex + return "pass" + + def get_handlers(self): + return { + 'boardsize' : self.handle_boardsize, + 'clear_board' : self.handle_clear_board, + 'komi' : self.handle_komi, + 'play' : self.handle_play, + 'genmove' : self.handle_genmove, + } + + +def make_player_engine(player): + """Return a Gtp_engine_protocol based on a specified player object. + + Actually returns a Test_gtp_engine_protocol. + + It has an additional 'player' attribute, which gives access to the + player object. + + """ + engine = Test_gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_commands(player.get_handlers()) + engine.player = player + return engine + +def get_test_player_engine(): + """Return a Gtp_engine_protocol based on a Test_player. + + Actually returns a Test_gtp_engine_protocol. + + It has an additional 'player' attribute, which gives access to the + Test_player. + + """ + return make_player_engine(Test_player()) + +def get_test_player_channel(): + """Return a Testing_gtp_channel connected to the test player engine.""" + engine = get_test_player_engine() + return gtp_controller_test_support.Testing_gtp_channel(engine) + + +## State reporter subprocess + +class State_reporter_fixture(test_framework.Fixture): + """Fixture for use with suprocess_state_reporter.py + + Attributes: + pathname -- pathname of the state reporter python script + cmd -- command list suitable for use with suprocess.Popen + devnull -- file open for writing to /dev/null + + """ + def __init__(self, tc): + self._pathname = os.path.abspath( + os.path.join(os.path.dirname(__file__), + "subprocess_state_reporter.py")) + self.cmd = ["python", self._pathname] + self.devnull = open(os.devnull, "w") + tc.addCleanup(self.devnull.close) + + +## Mock subprocess gtp channel + +class Mock_subprocess_gtp_channel( + gtp_controller_test_support.Testing_gtp_channel): + """Mock substitute for Subprocess_gtp_channel. + + This has the same construction interface as Subprocess_gtp_channel, but is + in fact a Testing_gtp_channel. + + This accepts the following 'command line arguments' in the 'command' list: + id= -- id to use in the channels registry + engine= -- look up engine in the engine registry + init= -- look up initialisation fn in the callback registry + fail=startup -- simulate exec failure + + By default, the underlying engine is a newly-created test player engine. + You can override this using 'engine=xxx'. + + If you want to get at the channel object after creating it, pass 'id=xxx' + and find it using the 'channels' class attribute. + + If you want to modify the returned channel object, pass 'init=xxx' and + register a callback function taking a channel parameter. + + Class attributes: + engine_registry -- map engine code -> Gtp_engine_protocol + callback_registry -- map callback code -> function + channels -- map id string -> Mock_subprocess_gtp_channel + + Instance attributes: + id -- string or None + requested_command -- list of strings + requested_stderr + requested_cwd + requested_env + + """ + engine_registry = {} + callback_registry = {} + channels = {} + + def __init__(self, command, stderr=None, cwd=None, env=None): + self.requested_command = command + self.requested_stderr = stderr + self.requested_cwd = cwd + self.requested_env = env + self.id = None + engine = None + callback = None + for arg in command[1:]: + key, eq, value = arg.partition("=") + if not eq: + raise SupporterError("Mock_subprocess_gtp_channel: " + "bad command-line argument: %s" % arg) + if key == 'id': + self.id = value + self.channels[value] = self + elif key == 'engine': + try: + engine = self.engine_registry[value] + except KeyError: + raise SupporterError( + "Mock_subprocess_gtp_channel: unregistered engine '%s'" + % value) + elif key == 'init': + try: + callback = self.callback_registry[value] + except KeyError: + raise SupporterError( + "Mock_subprocess_gtp_channel: unregistered init '%s'" + % value) + elif key == 'fail' and value == 'startup': + raise GtpChannelError("exec forced to fail") + else: + raise SupporterError("Mock_subprocess_gtp_channel: " + "bad command-line argument: %s" % arg) + + if engine is None: + engine = get_test_player_engine() + gtp_controller_test_support.Testing_gtp_channel.__init__(self, engine) + if callback is not None: + callback(self) + + +class Mock_subprocess_fixture(test_framework.Fixture): + """Fixture for using Mock_subprocess_gtp_channel. + + While this fixture is active, attempts to instantiate a + Subprocess_gtp_channel will produce a Testing_gtp_channel. + + """ + def __init__(self, tc): + self._patch() + tc.addCleanup(self._unpatch) + + def _patch(self): + self._sgc = gtp_controller.Subprocess_gtp_channel + gtp_controller.Subprocess_gtp_channel = Mock_subprocess_gtp_channel + + def _unpatch(self): + Mock_subprocess_gtp_channel.engine_registry.clear() + Mock_subprocess_gtp_channel.channels.clear() + gtp_controller.Subprocess_gtp_channel = self._sgc + + def register_engine(self, code, engine): + """Specify an engine for a mock subprocess channel to run. + + code -- string + engine -- Gtp_engine_protocol + + After this is called, attempts to instantiate a Subprocess_gtp_channel + with an 'engine=code' argument will return a Testing_gtp_channel based + on the specified engine. + + """ + Mock_subprocess_gtp_channel.engine_registry[code] = engine + + def register_init_callback(self, code, fn): + """Specify an initialisation callback for the mock subprocess channel. + + code -- string + fn -- function + + After this is called, attempts to instantiate a Subprocess_gtp_channel + with an 'init=code' argument will call the specified function, passing + the Testing_gtp_channel as its parameter. + + """ + Mock_subprocess_gtp_channel.callback_registry[code] = fn + + def get_channel(self, id): + """Retrieve a channel via its 'id' command-line argument.""" + return Mock_subprocess_gtp_channel.channels[id] diff --git a/gomill/gomill_tests/gtp_engine_test_support.py b/gomill/gomill_tests/gtp_engine_test_support.py new file mode 100644 index 0000000..4b0f642 --- /dev/null +++ b/gomill/gomill_tests/gtp_engine_test_support.py @@ -0,0 +1,51 @@ +"""Test support code for testing gomill GTP engines. + +(Which includes proxy engines.) + +""" + +def check_engine(tc, engine, command, args, expected, + expect_failure=False, expect_end=False, + expect_internal_error=False): + """Send a command to an engine and check its response. + + tc -- TestCase + engine -- Gtp_engine_protocol + command -- GTP command to send + args -- list of GTP arguments to send + expected -- expected response string (None to skip check) + expect_failure -- expect a GTP failure response + expect_end -- expect the engine to report 'end session' + expect_internal_error -- see below + + If the response isn't as expected, uses 'tc' to report this. + + If expect_internal_error is true, expect_failure is forced true, and the + check for expected (if specified) is that it's included in the response, + rather than equal to the response. + + """ + failure, response, end = engine.run_command(command, args) + if expect_internal_error: + expect_failure = True + if expect_failure: + tc.assertTrue(failure, + "unexpected GTP success response: %s" % response) + else: + tc.assertFalse(failure, + "unexpected GTP failure response: %s" % response) + if expect_internal_error: + tc.assertTrue(response.startswith("internal error\n"), response) + if expected is not None: + tc.assertTrue(expected in response, response) + elif expected is not None: + if command == "showboard": + tc.assertDiagramEqual(response, expected, + "showboard response not as expected") + else: + tc.assertEqual(response, expected, "GTP response not as expected") + if expect_end: + tc.assertTrue(end, "expected end-session not seen") + else: + tc.assertFalse(end, "unexpected end-session") + diff --git a/gomill/gomill_tests/gtp_engine_tests.py b/gomill/gomill_tests/gtp_engine_tests.py new file mode 100644 index 0000000..056f4b8 --- /dev/null +++ b/gomill/gomill_tests/gtp_engine_tests.py @@ -0,0 +1,56 @@ +"""Tests for gtp_engine.py""" + +from __future__ import with_statement + +from gomill import gtp_engine + +from gomill_tests import gomill_test_support +from gomill_tests import gtp_engine_test_support +from gomill_tests import test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +def test_engine(tc): + def handle_test(args): + if args: + return "args: " + " ".join(args) + else: + return "test response" + + engine = gtp_engine.Gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_command('test', handle_test) + + check_engine = gtp_engine_test_support.check_engine + check_engine(tc, engine, 'test', ['ab', 'cd'], "args: ab cd") + +def test_run_gtp_session(tc): + engine = gtp_engine.Gtp_engine_protocol() + engine.add_protocol_commands() + + stream = "known_command list_commands\nxyzzy\nquit\n" + command_pipe = test_support.Mock_reading_pipe(stream) + response_pipe = test_support.Mock_writing_pipe() + gtp_engine.run_gtp_session(engine, command_pipe, response_pipe) + tc.assertMultiLineEqual(response_pipe.getvalue(), + "= true\n\n? unknown command\n\n=\n\n") + command_pipe.close() + response_pipe.close() + +def test_run_gtp_session_broken_pipe(tc): + def break_pipe(args): + response_pipe.simulate_broken_pipe() + + engine = gtp_engine.Gtp_engine_protocol() + engine.add_protocol_commands() + engine.add_command("break", break_pipe) + + stream = "known_command list_commands\nbreak\nquit\n" + command_pipe = test_support.Mock_reading_pipe(stream) + response_pipe = test_support.Mock_writing_pipe() + with tc.assertRaises(gtp_engine.ControllerDisconnected) as ar: + gtp_engine.run_gtp_session(engine, command_pipe, response_pipe) + command_pipe.close() + response_pipe.close() + diff --git a/gomill/gomill_tests/gtp_game_tests.py b/gomill/gomill_tests/gtp_game_tests.py new file mode 100644 index 0000000..f7186a7 --- /dev/null +++ b/gomill/gomill_tests/gtp_game_tests.py @@ -0,0 +1,630 @@ +"""Tests for gtp_games.py""" + +import cPickle as pickle + +from gomill import gtp_controller +from gomill import gtp_games +from gomill import sgf +from gomill.common import format_vertex + +from gomill_tests import test_framework +from gomill_tests import gomill_test_support +from gomill_tests import gtp_controller_test_support +from gomill_tests import gtp_engine_fixtures +from gomill_tests.gtp_engine_fixtures import Programmed_player + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + for t in handicap_compensation_tests: + suite.addTest(Handicap_compensation_TestCase(*t)) + + +class Game_fixture(test_framework.Fixture): + """Fixture managing a Gtp_game. + + Instantiate with the player objects (defaults to a Test_player) and + optionally komi. + + attributes: + game -- Gtp_game + controller_b -- Gtp_controller + controller_w -- Gtp_controller + channel_b -- Testing_gtp_channel + channel_w -- Testing_gtp_channel + engine_b -- Test_gtp_engine_protocol + engine_w -- Test_gtp_engine_protocol + player_b -- player object + player_w -- player object + + """ + def __init__(self, tc, player_b=None, player_w=None, + komi=0.0, board_size=9): + self.tc = tc + game = gtp_games.Game(board_size, komi=komi) + game.set_player_code('b', 'one') + game.set_player_code('w', 'two') + if player_b is None: + player_b = gtp_engine_fixtures.Test_player() + if player_w is None: + player_w = gtp_engine_fixtures.Test_player() + engine_b = gtp_engine_fixtures.make_player_engine(player_b) + engine_w = gtp_engine_fixtures.make_player_engine(player_w) + channel_b = gtp_controller_test_support.Testing_gtp_channel(engine_b) + channel_w = gtp_controller_test_support.Testing_gtp_channel(engine_w) + controller_b = gtp_controller.Gtp_controller(channel_b, 'player one') + controller_w = gtp_controller.Gtp_controller(channel_w, 'player two') + game.set_player_controller('b', controller_b) + game.set_player_controller('w', controller_w) + self.game = game + self.controller_b = controller_b + self.controller_w = controller_w + self.channel_b = channel_b + self.channel_w = channel_w + self.engine_b = channel_b.engine + self.engine_w = channel_w.engine + self.player_b = channel_b.engine.player + self.player_w = channel_w.engine.player + + def check_moves(self, expected_moves): + """Check that the game's moves are as expected. + + expected_moves -- list of pairs (colour, vertex) + + """ + game_moves = [(colour, format_vertex(move)) + for (colour, move, comment) in self.game.moves] + self.tc.assertListEqual(game_moves, expected_moves) + + def run_score_test(self, b_score, w_score, allowed_scorers="bw"): + """Run a game and let the players score it. + + b_score, w_score -- string for final_score to return + + If b_score or w_score is None, the player won't implement final_score. + If b_score or w_score is an exception, the final_score will fail + + """ + def handle_final_score_b(args): + if b_score is Exception: + raise b_score + return b_score + def handle_final_score_w(args): + if w_score is Exception: + raise w_score + return w_score + if b_score is not None: + self.engine_b.add_command('final_score', handle_final_score_b) + if w_score is not None: + self.engine_w.add_command('final_score', handle_final_score_w) + for colour in allowed_scorers: + self.game.allow_scorer(colour) + self.game.ready() + self.game.run() + + def sgf_string(self): + return gomill_test_support.scrub_sgf( + self.game.make_sgf().serialise(wrap=None)) + + +def test_game(tc): + fx = Game_fixture(tc) + tc.assertDictEqual(fx.game.players, {'b' : 'one', 'w' : 'two'}) + tc.assertIs(fx.game.get_controller('b'), fx.controller_b) + tc.assertIs(fx.game.get_controller('w'), fx.controller_w) + fx.game.use_internal_scorer() + fx.game.ready() + tc.assertIsNone(fx.game.game_id) + tc.assertIsNone(fx.game.result) + fx.game.run() + fx.game.close_players() + tc.assertIsNone(fx.game.describe_late_errors()) + tc.assertDictEqual(fx.game.result.players, {'b' : 'one', 'w' : 'two'}) + tc.assertEqual(fx.game.result.player_b, 'one') + tc.assertEqual(fx.game.result.player_w, 'two') + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.result.losing_colour, 'w') + tc.assertEqual(fx.game.result.winning_player, 'one') + tc.assertEqual(fx.game.result.losing_player, 'two') + tc.assertEqual(fx.game.result.sgf_result, "B+18") + tc.assertFalse(fx.game.result.is_forfeit) + tc.assertIs(fx.game.result.is_jigo, False) + tc.assertIsNone(fx.game.result.detail) + tc.assertIsNone(fx.game.result.game_id) + tc.assertEqual(fx.game.result.describe(), "one beat two B+18") + result2 = pickle.loads(pickle.dumps(fx.game.result)) + tc.assertEqual(result2.describe(), "one beat two B+18") + tc.assertEqual(fx.game.describe_scoring(), "one beat two B+18") + tc.assertEqual(result2.player_b, 'one') + tc.assertEqual(result2.player_w, 'two') + tc.assertIs(result2.is_jigo, False) + tc.assertDictEqual(fx.game.result.cpu_times, {'one' : None, 'two' : None}) + tc.assertListEqual(fx.game.moves, [ + ('b', (0, 4), None), ('w', (0, 6), None), + ('b', (1, 4), None), ('w', (1, 6), None), + ('b', (2, 4), None), ('w', (2, 6), None), + ('b', (3, 4), None), ('w', (3, 6), None), + ('b', (4, 4), None), ('w', (4, 6), None), + ('b', (5, 4), None), ('w', (5, 6), None), + ('b', (6, 4), None), ('w', (6, 6), None), + ('b', (7, 4), None), ('w', (7, 6), None), + ('b', (8, 4), None), ('w', (8, 6), None), + ('b', None, None), ('w', None, None)]) + fx.check_moves([ + ('b', 'E1'), ('w', 'G1'), + ('b', 'E2'), ('w', 'G2'), + ('b', 'E3'), ('w', 'G3'), + ('b', 'E4'), ('w', 'G4'), + ('b', 'E5'), ('w', 'G5'), + ('b', 'E6'), ('w', 'G6'), + ('b', 'E7'), ('w', 'G7'), + ('b', 'E8'), ('w', 'G8'), + ('b', 'E9'), ('w', 'G9'), + ('b', 'pass'), ('w', 'pass'), + ]) + +def test_unscored_game(tc): + fx = Game_fixture(tc) + tc.assertDictEqual(fx.game.players, {'b' : 'one', 'w' : 'two'}) + tc.assertIs(fx.game.get_controller('b'), fx.controller_b) + tc.assertIs(fx.game.get_controller('w'), fx.controller_w) + fx.game.allow_scorer('b') # it can't score + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertIsNone(fx.game.describe_late_errors()) + tc.assertDictEqual(fx.game.result.players, {'b' : 'one', 'w' : 'two'}) + tc.assertIsNone(fx.game.result.winning_colour) + tc.assertIsNone(fx.game.result.losing_colour) + tc.assertIsNone(fx.game.result.winning_player) + tc.assertIsNone(fx.game.result.losing_player) + tc.assertEqual(fx.game.result.sgf_result, "?") + tc.assertFalse(fx.game.result.is_forfeit) + tc.assertIs(fx.game.result.is_jigo, False) + tc.assertEqual(fx.game.result.detail, "no score reported") + tc.assertEqual(fx.game.result.describe(), + "one vs two ? (no score reported)") + tc.assertEqual(fx.game.describe_scoring(), + "one vs two ? (no score reported)") + result2 = pickle.loads(pickle.dumps(fx.game.result)) + tc.assertEqual(result2.describe(), "one vs two ? (no score reported)") + tc.assertIs(result2.is_jigo, False) + +def test_jigo(tc): + fx = Game_fixture(tc, komi=18.0) + fx.game.use_internal_scorer() + fx.game.ready() + tc.assertIsNone(fx.game.result) + fx.game.run() + fx.game.close_players() + tc.assertIsNone(fx.game.describe_late_errors()) + tc.assertDictEqual(fx.game.result.players, {'b' : 'one', 'w' : 'two'}) + tc.assertEqual(fx.game.result.player_b, 'one') + tc.assertEqual(fx.game.result.player_w, 'two') + tc.assertEqual(fx.game.result.winning_colour, None) + tc.assertEqual(fx.game.result.losing_colour, None) + tc.assertEqual(fx.game.result.winning_player, None) + tc.assertEqual(fx.game.result.losing_player, None) + tc.assertEqual(fx.game.result.sgf_result, "0") + tc.assertIs(fx.game.result.is_forfeit, False) + tc.assertIs(fx.game.result.is_jigo, True) + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.describe(), "one vs two jigo") + tc.assertEqual(fx.game.describe_scoring(), "one vs two jigo") + result2 = pickle.loads(pickle.dumps(fx.game.result)) + tc.assertEqual(result2.describe(), "one vs two jigo") + tc.assertEqual(result2.player_b, 'one') + tc.assertEqual(result2.player_w, 'two') + tc.assertIs(result2.is_jigo, True) + +def test_players_score_agree(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3", "B+3.0") + tc.assertEqual(fx.game.result.sgf_result, "B+3") + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.describe_scoring(), "one beat two B+3") + +def test_players_score_agree_draw(tc): + fx = Game_fixture(tc) + fx.run_score_test("0", "0") + tc.assertEqual(fx.game.result.sgf_result, "0") + tc.assertIsNone(fx.game.result.detail) + tc.assertIsNone(fx.game.result.winning_colour) + tc.assertEqual(fx.game.describe_scoring(), "one vs two jigo") + +def test_players_score_disagree(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3.0", "W+4") + tc.assertEqual(fx.game.result.sgf_result, "?") + tc.assertEqual(fx.game.result.detail, "players disagreed") + tc.assertIsNone(fx.game.result.winning_colour) + tc.assertEqual(fx.game.describe_scoring(), + "one vs two ? (players disagreed)\n" + "one final_score: b+3.0\n" + "two final_score: W+4") + +def test_players_score_disagree_one_no_margin(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+", "W+4") + tc.assertEqual(fx.game.result.sgf_result, "?") + tc.assertEqual(fx.game.result.detail, "players disagreed") + tc.assertEqual(fx.game.describe_scoring(), + "one vs two ? (players disagreed)\n" + "one final_score: b+\n" + "two final_score: W+4") + +def test_players_score_disagree_one_jigo(tc): + fx = Game_fixture(tc) + fx.run_score_test("0", "W+4") + tc.assertEqual(fx.game.result.sgf_result, "?") + tc.assertEqual(fx.game.result.detail, "players disagreed") + tc.assertIsNone(fx.game.result.winning_colour) + tc.assertEqual(fx.game.describe_scoring(), + "one vs two ? (players disagreed)\n" + "one final_score: 0\n" + "two final_score: W+4") + +def test_players_score_disagree_equal_margin(tc): + # check equal margin in both directions doesn't confuse it + fx = Game_fixture(tc) + fx.run_score_test("b+4", "W+4") + tc.assertEqual(fx.game.result.sgf_result, "?") + tc.assertEqual(fx.game.result.detail, "players disagreed") + tc.assertIsNone(fx.game.result.winning_colour) + tc.assertEqual(fx.game.describe_scoring(), + "one vs two ? (players disagreed)\n" + "one final_score: b+4\n" + "two final_score: W+4") + +def test_players_score_one_unreliable(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3", "W+4", allowed_scorers="w") + tc.assertEqual(fx.game.result.sgf_result, "W+4") + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.describe_scoring(), "two beat one W+4") + +def test_players_score_one_cannot_score(tc): + fx = Game_fixture(tc) + fx.run_score_test(None, "W+4") + tc.assertEqual(fx.game.result.sgf_result, "W+4") + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.describe_scoring(), "two beat one W+4") + +def test_players_score_one_fails(tc): + fx = Game_fixture(tc) + fx.run_score_test(Exception, "W+4") + tc.assertEqual(fx.game.result.sgf_result, "W+4") + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.describe_scoring(), "two beat one W+4") + +def test_players_score_one_illformed(tc): + fx = Game_fixture(tc) + fx.run_score_test("black wins", "W+4.5") + tc.assertEqual(fx.game.result.sgf_result, "W+4.5") + tc.assertIsNone(fx.game.result.detail) + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.describe_scoring(), + "two beat one W+4.5\n" + "one final_score: black wins\n" + "two final_score: W+4.5") + +def test_players_score_agree_except_margin(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3", "B+4.0") + tc.assertEqual(fx.game.result.sgf_result, "B+") + tc.assertEqual(fx.game.result.detail, "unknown margin") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.describe_scoring(), + "one beat two B+ (unknown margin)\n" + "one final_score: b+3\n" + "two final_score: B+4.0") + +def test_players_score_agree_one_no_margin(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3", "B+") + tc.assertEqual(fx.game.result.sgf_result, "B+") + tc.assertEqual(fx.game.result.detail, "unknown margin") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.describe_scoring(), + "one beat two B+ (unknown margin)\n" + "one final_score: b+3\n" + "two final_score: B+") + +def test_players_score_agree_one_illformed_margin(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+3", "B+a") + tc.assertEqual(fx.game.result.sgf_result, "B+") + tc.assertEqual(fx.game.result.detail, "unknown margin") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.describe_scoring(), + "one beat two B+ (unknown margin)\n" + "one final_score: b+3\n" + "two final_score: B+a") + +def test_players_score_agree_margin_zero(tc): + fx = Game_fixture(tc) + fx.run_score_test("b+0", "B+0") + tc.assertEqual(fx.game.result.sgf_result, "B+") + tc.assertEqual(fx.game.result.detail, "unknown margin") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.describe_scoring(), + "one beat two B+ (unknown margin)\n" + "one final_score: b+0\n" + "two final_score: B+0") + + + +def test_claim(tc): + def handle_genmove_ex_b(args): + tc.assertIn('claim', args) + if fx.player_b.row_to_play < 3: + return fx.player_b.handle_genmove(args) + return "claim" + def handle_genmove_ex_w(args): + return "claim" + fx = Game_fixture(tc) + fx.engine_b.add_command('gomill-genmove_ex', handle_genmove_ex_b) + fx.engine_w.add_command('gomill-genmove_ex', handle_genmove_ex_w) + fx.game.use_internal_scorer() + fx.game.set_claim_allowed('b') + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "B+") + tc.assertEqual(fx.game.result.detail, "claim") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.result.winning_player, 'one') + tc.assertFalse(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.describe(), "one beat two B+ (claim)") + tc.assertEqual(fx.game.describe_scoring(), "one beat two B+ (claim)") + fx.check_moves([ + ('b', 'E1'), ('w', 'G1'), + ('b', 'E2'), ('w', 'G2'), + ('b', 'E3'), ('w', 'G3'), + ]) + +def test_forfeit_occupied_point(tc): + moves = [ + ('b', 'C3'), ('w', 'D3'), + ('b', 'D4'), ('w', 'D4'), # occupied point + ] + fx = Game_fixture(tc, Programmed_player(moves), Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "B+F") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.result.winning_player, 'one') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: two attempted move to occupied point d4") + tc.assertEqual(fx.game.result.describe(), + "one beat two B+F " + "(forfeit: two attempted move to occupied point d4)") + fx.check_moves(moves[:-1]) + +def test_forfeit_simple_ko(tc): + moves = [ + ('b', 'C5'), ('w', 'F5'), + ('b', 'D6'), ('w', 'E4'), + ('b', 'D4'), ('w', 'E6'), + ('b', 'E5'), ('w', 'D5'), + ('b', 'E5'), # ko violation + ] + fx = Game_fixture(tc, Programmed_player(moves), Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "W+F") + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.result.winning_player, 'two') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: one attempted move to ko-forbidden point e5") + fx.check_moves(moves[:-1]) + +def test_forfeit_illformed_move(tc): + moves = [ + ('b', 'C5'), ('w', 'F5'), + ('b', 'D6'), ('w', 'Z99'), # ill-formed move + ] + fx = Game_fixture(tc, Programmed_player(moves), Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "B+F") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.result.winning_player, 'one') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: two attempted ill-formed move z99") + fx.check_moves(moves[:-1]) + +def test_forfeit_genmove_fails(tc): + moves = [ + ('b', 'C5'), ('w', 'F5'), + ('b', 'fail'), # GTP failure response + ] + fx = Game_fixture(tc, Programmed_player(moves), Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "W+F") + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.result.winning_player, 'two') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: failure response from 'genmove b' to player one:\n" + "forced to fail") + fx.check_moves(moves[:-1]) + +def test_forfeit_rejected_as_illegal(tc): + moves = [ + ('b', 'C5'), ('w', 'F5'), + ('b', 'D6'), ('w', 'E4'), # will be rejected + ] + fx = Game_fixture(tc, + Programmed_player(moves, reject=('E4', 'illegal move')), + Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "B+F") + tc.assertEqual(fx.game.result.winning_colour, 'b') + tc.assertEqual(fx.game.result.winning_player, 'one') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: one claims move e4 is illegal") + fx.check_moves(moves[:-1]) + +def test_forfeit_play_failed(tc): + moves = [ + ('b', 'C5'), ('w', 'F5'), + ('b', 'D6'), ('w', 'E4'), # will be rejected + ] + fx = Game_fixture(tc, + Programmed_player(moves, reject=('E4', 'crash')), + Programmed_player(moves)) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "W+F") + tc.assertEqual(fx.game.result.winning_colour, 'w') + tc.assertEqual(fx.game.result.winning_player, 'two') + tc.assertTrue(fx.game.result.is_forfeit) + tc.assertEqual(fx.game.result.detail, + "forfeit: failure response from 'play w e4' to player one:\n" + "crash") + fx.check_moves(moves[:-1]) + +def test_same_player_code(tc): + game = gtp_games.Game(board_size=9, komi=0) + game.set_player_code('b', 'one') + tc.assertRaisesRegexp(ValueError, "player codes must be distinct", + game.set_player_code, 'w', 'one') + + +def test_make_sgf(tc): + fx = Game_fixture(tc) + fx.game.use_internal_scorer() + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertMultiLineEqual(fx.sgf_string(), """\ +(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]KM[0]RE[B+18]SZ[9];B[ei];W[gi];B[eh];W[gh];B[eg];W[gg];B[ef];W[gf];B[ee];W[ge];B[ed];W[gd];B[ec];W[gc];B[eb];W[gb];B[ea];W[ga];B[tt];C[one beat two B+18]W[tt]) +""") + tc.assertMultiLineEqual(gomill_test_support.scrub_sgf( + fx.game.make_sgf(game_end_message="zzzz").serialise(wrap=None)), """\ +(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]KM[0]RE[B+18]SZ[9];B[ei];W[gi];B[eh];W[gh];B[eg];W[gg];B[ef];W[gf];B[ee];W[ge];B[ed];W[gd];B[ec];W[gc];B[eb];W[gb];B[ea];W[ga];B[tt];C[one beat two B+18 + +zzzz]W[tt]) +""") + + +def test_game_id(tc): + fx = Game_fixture(tc) + fx.game.use_internal_scorer() + fx.game.set_game_id("gitest") + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.game_id, "gitest") + tc.assertMultiLineEqual(fx.sgf_string(), """\ +(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]GN[gitest]KM[0]RE[B+18]SZ[9];B[ei];W[gi];B[eh];W[gh];B[eg];W[gg];B[ef];W[gf];B[ee];W[ge];B[ed];W[gd];B[ec];W[gc];B[eb];W[gb];B[ea];W[ga];B[tt];C[one beat two B+18]W[tt]) +""") + +def test_explain_last_move(tc): + counter = [0] + def handle_explain_last_move(args): + counter[0] += 1 + return "EX%d" % counter[0] + fx = Game_fixture(tc) + fx.engine_b.add_command('gomill-explain_last_move', + handle_explain_last_move) + fx.game.ready() + fx.game.run() + fx.game.close_players() + tc.assertMultiLineEqual(fx.sgf_string(), """\ +(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]KM[0]RE[?]SZ[9];B[ei]C[EX1];W[gi];B[eh]C[EX2];W[gh];B[eg]C[EX3];W[gg];B[ef]C[EX4];W[gf];B[ee]C[EX5];W[ge];B[ed]C[EX6];W[gd];B[ec]C[EX7];W[gc];B[eb]C[EX8];W[gb];B[ea]C[EX9];W[ga];B[tt]C[EX10];C[one vs two ? (no score reported)]W[tt]) +""") + + +def test_fixed_handicap(tc): + fh_calls = [] + def handle_fixed_handicap(args): + fh_calls.append(args[0]) + return "C3 G7 C7" + fx = Game_fixture(tc) + fx.engine_b.add_command('fixed_handicap', handle_fixed_handicap) + fx.engine_w.add_command('fixed_handicap', handle_fixed_handicap) + fx.game.ready() + fx.game.set_handicap(3, is_free=False) + tc.assertEqual(fh_calls, ["3", "3"]) + fx.game.run() + fx.game.close_players() + tc.assertEqual(fx.game.result.sgf_result, "B+F") + tc.assertEqual(fx.game.result.detail, + "forfeit: two attempted move to occupied point g7") + fx.check_moves([ + ('w', 'G1'), ('b', 'E1'), + ('w', 'G2'), ('b', 'E2'), + ('w', 'G3'), ('b', 'E3'), + ('w', 'G4'), ('b', 'E4'), + ('w', 'G5'), ('b', 'E5'), + ('w', 'G6'), ('b', 'E6'), + ]) + tc.assertMultiLineEqual(fx.sgf_string(), """\ +(;FF[4]AB[cc][cg][gc]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]HA[3]KM[0]RE[B+F]SZ[9];W[gi];B[ei];W[gh];B[eh];W[gg];B[eg];W[gf];B[ef];W[ge];B[ee];W[gd];B[ed]C[one beat two B+F (forfeit: two attempted move to occupied point g7)]) +""") + +def test_fixed_handicap_bad_engine(tc): + fh_calls = [] + def handle_fixed_handicap_good(args): + fh_calls.append(args[0]) + return "g7 c7 c3" + def handle_fixed_handicap_bad(args): + fh_calls.append(args[0]) + return "C3 G3 C7" # Should be G7, not G3 + fx = Game_fixture(tc) + fx.engine_b.add_command('fixed_handicap', handle_fixed_handicap_good) + fx.engine_w.add_command('fixed_handicap', handle_fixed_handicap_bad) + fx.game.ready() + tc.assertRaisesRegexp( + gtp_controller.BadGtpResponse, + "^bad response from fixed_handicap command to two: C3 G3 C7$", + fx.game.set_handicap, 3, is_free=False) + + +handicap_compensation_tests = [ + # test code, handicap_compensation, result + ('no', 'no', "B+53"), + ('full', 'full', "B+50"), + ('short', 'short', "B+51"), + ] + +class Handicap_compensation_TestCase( + gomill_test_support.Gomill_ParameterisedTestCase): + test_name = "test_handicap_compensation" + parameter_names = ('hc', 'result') + + def runTest(self): + def handle_fixed_handicap(args): + return "D4 K10 D10" + fx = Game_fixture(self, board_size=13) + fx.engine_b.add_command('fixed_handicap', handle_fixed_handicap) + fx.engine_w.add_command('fixed_handicap', handle_fixed_handicap) + fx.game.use_internal_scorer(handicap_compensation=self.hc) + fx.game.set_handicap(3, is_free=False) + fx.game.ready() + fx.game.run() + fx.game.close_players() + self.assertEqual(fx.game.result.sgf_result, self.result) diff --git a/gomill/gomill_tests/gtp_proxy_tests.py b/gomill/gomill_tests/gtp_proxy_tests.py new file mode 100644 index 0000000..f3ba2da --- /dev/null +++ b/gomill/gomill_tests/gtp_proxy_tests.py @@ -0,0 +1,242 @@ +"""Tests for gtp_proxy.py""" + +from __future__ import with_statement + +from gomill import gtp_controller +from gomill import gtp_proxy +from gomill.gtp_engine import GtpError, GtpFatalError +from gomill.gtp_controller import ( + GtpChannelError, GtpProtocolError, GtpTransportError, GtpChannelClosed, + BadGtpResponse, Gtp_controller) +from gomill.gtp_proxy import BackEndError + +from gomill_tests import test_framework +from gomill_tests import gomill_test_support +from gomill_tests import gtp_controller_test_support +from gomill_tests import gtp_engine_fixtures +from gomill_tests import gtp_engine_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +class Proxy_fixture(test_framework.Fixture): + """Fixture managing a Gtp_proxy with the test engine as its back-end. + + attributes: + proxy -- Gtp_proxy + controller -- Gtp_controller + channel -- Testing_gtp_channel (like get_test_channel()) + engine -- the proxy engine + underlying_engine -- the underlying test engine (like get_test_engine()) + commands_handled -- from the underlying Test_gtp_engine_protocol + + """ + def __init__(self, tc): + self.tc = tc + self.channel = gtp_engine_fixtures.get_test_channel() + self.underlying_engine = self.channel.engine + self.controller = gtp_controller.Gtp_controller( + self.channel, 'testbackend') + self.proxy = gtp_proxy.Gtp_proxy() + self.proxy.set_back_end_controller(self.controller) + self.engine = self.proxy.engine + self.commands_handled = self.underlying_engine.commands_handled + + def check_command(self, *args, **kwargs): + """Send a command to the proxy engine and check its response. + + (This is testing the proxy engine, not the underlying engine.) + + parameters as for gtp_engine_test_support.check_engine() + + """ + gtp_engine_test_support.check_engine( + self.tc, self.engine, *args, **kwargs) + + +def test_proxy(tc): + fx = Proxy_fixture(tc) + fx.check_command('test', ['ab', 'cd'], "args: ab cd") + fx.proxy.close() + tc.assertEqual( + fx.commands_handled, + [('list_commands', []), ('test', ['ab', 'cd']), ('quit', [])]) + tc.assertTrue(fx.controller.channel.is_closed) + +def test_close_after_quit(tc): + fx = Proxy_fixture(tc) + fx.check_command('quit', [], "", expect_end=True) + fx.proxy.close() + tc.assertEqual( + fx.commands_handled, + [('list_commands', []), ('quit', [])]) + tc.assertTrue(fx.channel.is_closed) + +def test_list_commands(tc): + fx = Proxy_fixture(tc) + tc.assertListEqual( + fx.engine.list_commands(), + ['error', 'fatal', 'gomill-passthrough', 'known_command', + 'list_commands', 'multiline', 'protocol_version', 'quit', 'test']) + fx.proxy.close() + +def test_back_end_has_command(tc): + fx = Proxy_fixture(tc) + tc.assertTrue(fx.proxy.back_end_has_command('test')) + tc.assertFalse(fx.proxy.back_end_has_command('xyzzy')) + tc.assertFalse(fx.proxy.back_end_has_command('gomill-passthrough')) + +def test_passthrough(tc): + fx = Proxy_fixture(tc) + fx.check_command('known_command', ['gomill-passthrough'], "true") + fx.check_command('gomill-passthrough', ['test', 'ab', 'cd'], "args: ab cd") + fx.check_command( + 'gomill-passthrough', ['known_command', 'gomill-passthrough'], "false") + fx.check_command('gomill-passthrough', [], + "invalid arguments", expect_failure=True) + tc.assertEqual( + fx.commands_handled, + [('list_commands', []), ('test', ['ab', 'cd']), + ('known_command', ['gomill-passthrough'])]) + +def test_pass_command(tc): + fx = Proxy_fixture(tc) + tc.assertEqual(fx.proxy.pass_command("test", ["ab", "cd"]), "args: ab cd") + with tc.assertRaises(BadGtpResponse) as ar: + fx.proxy.pass_command("error", []) + tc.assertEqual(ar.exception.gtp_error_message, "normal error") + tc.assertEqual(str(ar.exception), + "failure response from 'error' to testbackend:\n" + "normal error") + +def test_pass_command_with_channel_error(tc): + fx = Proxy_fixture(tc) + fx.channel.fail_next_command = True + with tc.assertRaises(BackEndError) as ar: + fx.proxy.pass_command("test", []) + tc.assertEqual(str(ar.exception), + "transport error sending 'test' to testbackend:\n" + "forced failure for send_command_line") + tc.assertIsInstance(ar.exception.cause, GtpTransportError) + fx.proxy.close() + tc.assertEqual(fx.commands_handled, [('list_commands', [])]) + +def test_handle_command(tc): + def handle_xyzzy(args): + if args and args[0] == "error": + return fx.proxy.handle_command("error", []) + else: + return fx.proxy.handle_command("test", ["nothing", "happens"]) + fx = Proxy_fixture(tc) + fx.engine.add_command("xyzzy", handle_xyzzy) + fx.check_command('xyzzy', [], "args: nothing happens") + fx.check_command('xyzzy', ['error'], + "normal error", expect_failure=True) + +def test_handle_command_with_channel_error(tc): + def handle_xyzzy(args): + return fx.proxy.handle_command("test", []) + fx = Proxy_fixture(tc) + fx.engine.add_command("xyzzy", handle_xyzzy) + fx.channel.fail_next_command = True + fx.check_command('xyzzy', [], + "transport error sending 'test' to testbackend:\n" + "forced failure for send_command_line", + expect_failure=True, expect_end=True) + fx.proxy.close() + tc.assertEqual(fx.commands_handled, [('list_commands', [])]) + +def test_back_end_goes_away(tc): + fx = Proxy_fixture(tc) + tc.assertEqual(fx.proxy.pass_command("quit", []), "") + fx.check_command('test', ['ab', 'cd'], + "error sending 'test ab cd' to testbackend:\n" + "engine has closed the command channel", + expect_failure=True, expect_end=True) + +def test_close_with_errors(tc): + fx = Proxy_fixture(tc) + fx.channel.fail_next_command = True + with tc.assertRaises(BackEndError) as ar: + fx.proxy.close() + tc.assertEqual(str(ar.exception), + "transport error sending 'quit' to testbackend:\n" + "forced failure for send_command_line") + tc.assertTrue(fx.channel.is_closed) + +def test_quit_ignores_already_closed(tc): + fx = Proxy_fixture(tc) + tc.assertEqual(fx.proxy.pass_command("quit", []), "") + fx.check_command('quit', [], "", expect_end=True) + fx.proxy.close() + tc.assertEqual(fx.commands_handled, + [('list_commands', []), ('quit', [])]) + +def test_quit_with_failure_response(tc): + fx = Proxy_fixture(tc) + fx.underlying_engine.force_error("quit") + fx.check_command('quit', [], None, + expect_failure=True, expect_end=True) + fx.proxy.close() + tc.assertEqual(fx.commands_handled, + [('list_commands', []), ('quit', [])]) + +def test_quit_with_channel_error(tc): + fx = Proxy_fixture(tc) + fx.channel.fail_next_command = True + fx.check_command('quit', [], + "transport error sending 'quit' to testbackend:\n" + "forced failure for send_command_line", + expect_failure=True, expect_end=True) + fx.proxy.close() + tc.assertEqual(fx.commands_handled, [('list_commands', [])]) + +def test_nontgtp_backend(tc): + channel = gtp_controller_test_support.Preprogrammed_gtp_channel( + "Usage: randomprogram [options]\n\nOptions:\n" + "--help show this help message and exit\n") + controller = gtp_controller.Gtp_controller(channel, 'testbackend') + proxy = gtp_proxy.Gtp_proxy() + with tc.assertRaises(BackEndError) as ar: + proxy.set_back_end_controller(controller) + tc.assertEqual(str(ar.exception), + "GTP protocol error reading response to first command " + "(list_commands) from testbackend:\n" + "engine isn't speaking GTP: first byte is 'U'") + tc.assertIsInstance(ar.exception.cause, GtpProtocolError) + proxy.close() + +def test_error_from_list_commands(tc): + channel = gtp_engine_fixtures.get_test_channel() + channel.engine.force_error("list_commands") + controller = gtp_controller.Gtp_controller(channel, 'testbackend') + proxy = gtp_proxy.Gtp_proxy() + with tc.assertRaises(BackEndError) as ar: + proxy.set_back_end_controller(controller) + tc.assertEqual(str(ar.exception), + "failure response from first command " + "(list_commands) to testbackend:\n" + "handler forced to fail") + tc.assertIsInstance(ar.exception.cause, BadGtpResponse) + proxy.close() + + +def test_set_back_end_subprocess(tc): + fx = gtp_engine_fixtures.State_reporter_fixture(tc) + proxy = gtp_proxy.Gtp_proxy() + # the state-report will be taken as the response to list_commands + proxy.set_back_end_subprocess(fx.cmd, stderr=fx.devnull) + proxy.expect_back_end_exit() + proxy.close() + +def test_set_back_end_subprocess_nonexistent_program(tc): + proxy = gtp_proxy.Gtp_proxy() + with tc.assertRaises(BackEndError) as ar: + proxy.set_back_end_subprocess("/nonexistent/program") + tc.assertEqual(str(ar.exception), + "can't launch back end command\n" + "[Errno 2] No such file or directory") + tc.assertIsInstance(ar.exception.cause, GtpChannelError) + # check it's safe to close when the controller was never set + proxy.close() diff --git a/gomill/gomill_tests/gtp_state_test_support.py b/gomill/gomill_tests/gtp_state_test_support.py new file mode 100644 index 0000000..a416c71 --- /dev/null +++ b/gomill/gomill_tests/gtp_state_test_support.py @@ -0,0 +1,106 @@ +"""Support code for testing gtp_states.""" + +from gomill import gtp_states +from gomill.common import * + +class Player(object): + """Player (stateful move generator) for testing gtp_states. + + Public attributes: + last_game_state -- the Game_state from the last genmove-like command + + """ + + def __init__(self): + self.next_move = None + self.next_comment = None + self.next_cookie = None + self.last_game_state = None + self.resign_next_move = False + + def set_next_move(self, vertex, comment=None, cookie=None): + """Specify what to return from the next genmove-like command.""" + self.next_move = move_from_vertex(vertex, 19) + self.next_comment = comment + self.next_cookie = cookie + + def set_next_move_resign(self): + self.resign_next_move = True + + def genmove(self, game_state, player): + """Move generator returns points from the move list. + + game_state -- gtp_states.Game_state + player -- 'b' or 'w' + + """ + self.last_game_state = game_state + # Freeze the move_history as we saw it + self.last_game_state.move_history = self.last_game_state.move_history[:] + + result = gtp_states.Move_generator_result() + if self.resign_next_move: + result.resign = True + elif self.next_move is not None: + result.move = self.next_move + else: + result.pass_move = True + if self.next_comment is not None: + result.comments = self.next_comment + if self.next_cookie is not None: + result.cookie = self.next_cookie + self.next_move = None + self.next_comment = None + self.next_cookie = None + self.resign_next_move = False + return result + +class Testing_gtp_state(gtp_states.Gtp_state): + """Variant of Gtp_state suitable for use in tests. + + This doesn't read from or write to the filesystem. + + """ + def __init__(self, *args, **kwargs): + super(Testing_gtp_state, self).__init__(*args, **kwargs) + self._file_contents = {} + + def _register_file(self, pathname, contents): + self._file_contents[pathname] = contents + + def _load_file(self, pathname): + try: + return self._file_contents[pathname] + except KeyError: + raise EnvironmentError("unknown file: %s" % pathname) + + def _save_file(self, pathname, contents): + if pathname == "force_fail": + open("/nonexistent_directory/foo.sgf", "w") + self._file_contents[pathname] = contents + + def _choose_free_handicap_moves(self, number_of_stones): + """Implementation of place_free_handicap. + + Returns for a given number of stones: + 2 -- A1 A2 A3 (too many stones) + 4 -- A1 A2 A3 pass (pass isn't permitted) + 5 -- A1 A2 A3 A4 A5 (which is ok) + 6 -- A1 A2 A3 A4 A5 A1 (repeated point) + 8 -- not even the right result type + otherwise Gtp_state default (which is to use the fixed handicap points) + + """ + if number_of_stones == 2: + return ((0, 0), (1, 0), (2, 0)) + elif number_of_stones == 4: + return ((0, 0), (1, 0), (2, 0), None) + elif number_of_stones == 5: + return ((0, 0), (1, 0), (2, 0), (3, 0), (4, 0)) + elif number_of_stones == 6: + return ((0, 0), (1, 0), (2, 0), (3, 0), (4, 0), (0, 0)) + elif number_of_stones == 8: + return "nonsense" + else: + return super(Testing_gtp_state, self).\ + _choose_free_handicap_moves(number_of_stones) diff --git a/gomill/gomill_tests/gtp_state_tests.py b/gomill/gomill_tests/gtp_state_tests.py new file mode 100644 index 0000000..df5ea52 --- /dev/null +++ b/gomill/gomill_tests/gtp_state_tests.py @@ -0,0 +1,581 @@ +"""Tests for gtp_state.py.""" + +from textwrap import dedent + +from gomill import boards +from gomill import gtp_engine +from gomill import gtp_states +from gomill.common import format_vertex + +from gomill_tests import test_framework +from gomill_tests import gomill_test_support +from gomill_tests import gtp_engine_test_support +from gomill_tests import gtp_state_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +class Gtp_state_fixture(test_framework.Fixture): + """Fixture for managing a Gtp_state. + + The move generator comes from gtp_state_test_support.Player + + Adds a type equality function for History_move. + + """ + def __init__(self, tc): + self.tc = tc + self.player = gtp_state_test_support.Player() + self.gtp_state = gtp_state_test_support.Testing_gtp_state( + move_generator=self.player.genmove, + acceptable_sizes=(9, 11, 13, 19)) + self.engine = gtp_engine.Gtp_engine_protocol() + self.engine.add_protocol_commands() + self.engine.add_commands(self.gtp_state.get_handlers()) + self.tc.addTypeEqualityFunc( + gtp_states.History_move, self.assertHistoryMoveEqual) + + def assertHistoryMoveEqual(self, hm1, hm2, msg=None): + t1 = (hm1.colour, hm1.move, hm1.comments, hm1.cookie) + t2 = (hm2.colour, hm2.move, hm2.comments, hm2.cookie) + self.tc.assertEqual(t1, t2, "History_moves differ") + + def check_command(self, *args, **kwargs): + """Check a single GTP command. + + parameters as for gtp_engine_test_support.check_engine() + + """ + gtp_engine_test_support.check_engine( + self.tc, self.engine, *args, **kwargs) + + def check_board_empty_9(self): + self.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 . . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + +def test_gtp_state(tc): + fx = Gtp_state_fixture(tc) + + fx.check_command('nonsense', [''], "unknown command", + expect_failure=True) + fx.check_command('protocol_version', [''], "2") + + fx.player.set_next_move("A3", "preprogrammed move 0") + fx.check_command('genmove', ['B'], "A3") + game_state = fx.player.last_game_state + # default board size is min(acceptable_sizes) + tc.assertEqual(game_state.size, 9) + b = boards.Board(9) + b.play(2, 0, 'b') + tc.assertEqual(game_state.board, b) + tc.assertEqual(game_state.komi, 0.0) + tc.assertEqual(game_state.history_base, boards.Board(9)) + tc.assertEqual(game_state.move_history, []) + tc.assertIsNone(game_state.ko_point) + tc.assertIsNone(game_state.handicap) + tc.assertIs(game_state.for_regression, False) + tc.assertIsNone(game_state.time_settings) + tc.assertIsNone(game_state.time_remaining) + tc.assertIsNone(game_state.canadian_stones_remaining) + fx.check_command('gomill-explain_last_move', [], "preprogrammed move 0") + + fx.check_command('play', ['W', 'A4'], "") + fx.check_command('komi', ['5.5'], "") + fx.player.set_next_move("C9") + fx.check_command('genmove', ['B'], "C9") + game_state = fx.player.last_game_state + tc.assertEqual(game_state.komi, 5.5) + tc.assertEqual(game_state.history_base, boards.Board(9)) + tc.assertEqual(len(game_state.move_history), 2) + tc.assertEqual(game_state.move_history[0], + gtp_states.History_move('b', (2, 0), "preprogrammed move 0")) + tc.assertEqual(game_state.move_history[1], + gtp_states.History_move('w', (3, 0))) + + fx.check_command('genmove', ['B'], "pass") + fx.check_command('gomill-explain_last_move', [], "") + fx.check_command('genmove', ['W'], "pass") + fx.check_command('showboard', [], dedent(""" + 9 . . # . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 o . . . . . . . . + 3 # . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + + fx.player.set_next_move_resign() + fx.check_command('genmove', ['B'], "resign") + fx.check_command('quit', [''], "", expect_end=True) + + +def test_clear_board_and_boardsize(tc): + fx = Gtp_state_fixture(tc) + fx.check_command('play', ['W', 'A4'], "") + fx.check_command('boardsize', ['7'], "unacceptable size", + expect_failure=True) + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 o . . . . . . . . + 3 . . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('clear_board', [], "") + fx.check_board_empty_9() + fx.check_command('play', ['W', 'A4'], "") + fx.check_command('boardsize', ['11'], "") + fx.check_command('showboard', [], dedent(""" + 11 . . . . . . . . . . . + 10 . . . . . . . . . . . + 9 . . . . . . . . . . . + 8 . . . . . . . . . . . + 7 . . . . . . . . . . . + 6 . . . . . . . . . . . + 5 . . . . . . . . . . . + 4 . . . . . . . . . . . + 3 . . . . . . . . . . . + 2 . . . . . . . . . . . + 1 . . . . . . . . . . . + A B C D E F G H J K L""")) + +def test_play(tc): + fx = Gtp_state_fixture(tc) + fx.check_command('play', ['B', "E5"], "") + fx.check_command('play', ['w', "e4"], "") + fx.check_command('play', [], "invalid arguments", expect_failure=True) + fx.check_command('play', ['B'], "invalid arguments", expect_failure=True) + # additional arguments are ignored (following gnugo) + fx.check_command('play', ['B', "F4", "E5"], "") + fx.check_command('play', ['white', "f5"], "") + fx.check_command('play', ['W', "K3"], "vertex is off board: 'k3'", + expect_failure=True) + fx.check_command('play', ['X', "A4"], "invalid colour: 'X'", + expect_failure=True) + fx.check_command('play', ['BLACK', "e4"], "illegal move", + expect_failure=True) + fx.check_command('play', ['B', "pass"], "") + fx.check_command('play', ['W', "PASS"], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . # o . . . + 4 . . . . o # . . . + 3 . . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + +def test_komi(tc): + fx = Gtp_state_fixture(tc) + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, 0.0) + fx.check_command('komi', ['1'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, 1.0) + fx.check_command('komi', ['1.0'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, 1.0) + fx.check_command('komi', ['7.5'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, 7.5) + fx.check_command('komi', ['-3.5'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, -3.5) + fx.check_command('komi', ['20000'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, 625.0) + fx.check_command('komi', ['-20000'], "") + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.komi, -625.0) + fx.check_command('komi', ['nonsense'], "invalid float: 'nonsense'", + expect_failure=True) + fx.check_command('komi', ['NaN'], "invalid float: 'NaN'", + expect_failure=True) + fx.check_command('komi', ['inf'], "invalid float: 'inf'", + expect_failure=True) + fx.check_command('komi', ['-1e400'], "invalid float: '-1e400'", + expect_failure=True) + +def test_undo(tc): + fx = Gtp_state_fixture(tc) + fx.player.set_next_move("A3", "preprogrammed move A3") + fx.check_command('genmove', ['B'], "A3") + fx.check_command('gomill-explain_last_move', [], "preprogrammed move A3") + fx.check_command('play', ['W', 'A4'], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 o . . . . . . . . + 3 # . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('undo', [], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 # . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.player.set_next_move("D4", "preprogrammed move D4") + fx.check_command('genmove', ['W'], "D4") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . o . . . . . + 3 # . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('gomill-explain_last_move', [], "preprogrammed move D4") + fx.check_command('undo', [], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 # . . . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('gomill-explain_last_move', [], "preprogrammed move A3") + fx.check_command('undo', [], "") + fx.check_board_empty_9() + fx.check_command('gomill-explain_last_move', [], "") + fx.check_command('undo', [], "cannot undo", expect_failure=True) + +def test_fixed_handicap(tc): + fx = Gtp_state_fixture(tc) + fx.check_command('fixed_handicap', ['3'], "C3 G7 C7") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . # . . . # . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 . . # . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.handicap, 3) + fx.check_command('boardsize', ['19'], "") + fx.check_command('fixed_handicap', ['7'], "D4 Q16 D16 Q4 D10 Q10 K10") + fx.check_command('fixed_handicap', ['7'], "board not empty", + expect_failure=True) + fx.check_command('boardsize', ['9'], "") + fx.check_command('play', ['B', 'B2'], "") + fx.check_command('fixed_handicap', ['2'], "board not empty", + expect_failure=True) + fx.check_command('clear_board', [], "") + fx.check_command('fixed_handicap', ['0'], "invalid number of stones", + expect_failure=True) + fx.check_command('fixed_handicap', ['1'], "invalid number of stones", + expect_failure=True) + fx.check_command('fixed_handicap', ['10'], "invalid number of stones", + expect_failure=True) + fx.check_command('fixed_handicap', ['2.5'], "invalid int: '2.5'", + expect_failure=True) + fx.check_command('fixed_handicap', [], "invalid arguments", + expect_failure=True) + +def test_place_free_handicap(tc): + # See gtp_state_test_support.Testing_gtp_state for description of the choice + # of points. + fx = Gtp_state_fixture(tc) + fx.check_command('place_free_handicap', ['3'], "C3 G7 C7") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . # . . . # . . + 6 . . . . . . . . . + 5 . . . . . . . . . + 4 . . . . . . . . . + 3 . . # . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.handicap, 3) + fx.check_command('boardsize', ['19'], "") + fx.check_command('place_free_handicap', ['7'], "D4 Q16 D16 Q4 D10 Q10 K10") + fx.check_command('place_free_handicap', ['7'], "board not empty", + expect_failure=True) + fx.check_command('boardsize', ['9'], "") + fx.check_command('play', ['B', 'B2'], "") + fx.check_command('place_free_handicap', ['2'], "board not empty", + expect_failure=True) + fx.check_command('clear_board', [], "") + fx.check_command('place_free_handicap', ['0'], "invalid number of stones", + expect_failure=True) + fx.check_command('place_free_handicap', ['1'], "invalid number of stones", + expect_failure=True) + fx.check_command('place_free_handicap', ['2.5'], "invalid int: '2.5'", + expect_failure=True) + fx.check_command('place_free_handicap', [], "invalid arguments", + expect_failure=True) + fx.check_command('place_free_handicap', ['10'], + "C3 G7 C7 G3 C5 G5 E3 E7 E5") + fx.check_command('clear_board', [''], "") + fx.check_command('place_free_handicap', ['5'], + "A1 A2 A3 A4 A5") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 # . . . . . . . . + 4 # . . . . . . . . + 3 # . . . . . . . . + 2 # . . . . . . . . + 1 # . . . . . . . . + A B C D E F G H J""")) + fx.check_command('clear_board', [''], "") + fx.check_command('place_free_handicap', ['6'], + "invalid result from move generator: A1,A2,A3,A4,A5,A1", + expect_failure=True) + fx.check_board_empty_9() + fx.check_command('place_free_handicap', ['2'], + "invalid result from move generator: A1,A2,A3", + expect_failure=True) + fx.check_board_empty_9() + fx.check_command('place_free_handicap', ['4'], + "invalid result from move generator: A1,A2,A3,pass", + expect_failure=True) + fx.check_board_empty_9() + fx.check_command('place_free_handicap', ['8'], + "ValueError: need more than 1 value to unpack", + expect_internal_error=True) + fx.check_board_empty_9() + +def test_set_free_handicap(tc): + fx = Gtp_state_fixture(tc) + fx.check_command('set_free_handicap', ["C3", "E5", "C7"], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . # . . . . . . + 6 . . . . . . . . . + 5 . . . . # . . . . + 4 . . . . . . . . . + 3 . . # . . . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('genmove', ['B'], "pass") + tc.assertEqual(fx.player.last_game_state.handicap, 3) + fx.check_command('boardsize', ['9'], "") + fx.check_command('play', ['B', 'B2'], "") + fx.check_command('set_free_handicap', ["C3", "E5"], "board not empty", + expect_failure=True) + fx.check_command('clear_board', [], "") + fx.check_command('set_free_handicap', ["C3"], "invalid number of stones", + expect_failure=True) + fx.check_command('set_free_handicap', [], "invalid number of stones", + expect_failure=True) + all_points = [format_vertex((i, j)) for i in range(9) for j in range(9)] + fx.check_command('set_free_handicap', all_points, + "invalid number of stones", expect_failure=True) + fx.check_command('set_free_handicap', ["C3", "asdasd"], + "invalid vertex: 'asdasd'", expect_failure=True) + fx.check_board_empty_9() + fx.check_command('set_free_handicap', ["C3", "pass"], + "'pass' not permitted", expect_failure=True) + fx.check_board_empty_9() + fx.check_command('set_free_handicap', ["C3", "E5", "C3"], + "engine error: C3 is occupied", expect_failure=True) + fx.check_board_empty_9() + + +def test_loadsgf(tc): + fx = Gtp_state_fixture(tc) + fx.gtp_state._register_file("invalid.sgf", "non-SGF data") + fx.gtp_state._register_file( + "test1.sgf", + "(;SZ[9];B[ee];W[eg];B[dg];W[dh];B[df];W[fh];B[];W[])") + fx.gtp_state._register_file( + "test2.sgf", + "(;SZ[9]AB[fe:ff]AW[gf:gg]PL[W];W[eh];B[ge])") + fx.check_command('loadsgf', ["unknown.sgf"], + "cannot load file", expect_failure=True) + fx.check_command('loadsgf', ["invalid.sgf"], + "cannot load file", expect_failure=True) + fx.check_command('loadsgf', ["test1.sgf"], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . # . . . . + 4 . . . # . . . . . + 3 . . . # o . . . . + 2 . . . o . o . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('loadsgf', ["test1.sgf", "4"], "") + # position _before_ move 4 + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . # . . . . + 4 . . . . . . . . . + 3 . . . # o . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('undo', [], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . # . . . . + 4 . . . . . . . . . + 3 . . . . o . . . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('loadsgf', ["test2.sgf"], "") + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . # # . . + 4 . . . . . # o . . + 3 . . . . . . o . . + 2 . . . . o . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + fx.check_command('undo', [], "") + fx.check_command('undo', [], "") + fx.check_command('undo', [], "cannot undo", expect_failure=True) + fx.check_command('showboard', [], dedent(""" + 9 . . . . . . . . . + 8 . . . . . . . . . + 7 . . . . . . . . . + 6 . . . . . . . . . + 5 . . . . . # . . . + 4 . . . . . # o . . + 3 . . . . . . o . . + 2 . . . . . . . . . + 1 . . . . . . . . . + A B C D E F G H J""")) + +def test_savesgf(tc): + scrub_sgf = gomill_test_support.scrub_sgf + + fx = Gtp_state_fixture(tc) + fx.check_command("play", ['B', 'D4'], "") + fx.player.set_next_move("C3", "preprogrammed move C3") + fx.check_command('genmove', ['W'], "C3") + fx.check_command('gomill-savesgf', ['out1.sgf'], "") + tc.assertEqual( + scrub_sgf(fx.gtp_state._load_file('out1.sgf')).replace("\n", ""), + "(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]KM[0]" + "SZ[9];B[df];C[preprogrammed move C3]W[cg])") + fx.check_command( + 'gomill-savesgf', + ['out2.sgf', "PB=testplayer", "PW=GNU\_Go:3.8", "RE=W+3.5"], + "") + tc.assertEqual( + scrub_sgf(fx.gtp_state._load_file('out2.sgf')).replace("\n", ""), + "(;FF[4]AP[gomill:VER]CA[UTF-8]DT[***]GM[1]KM[0]" + "PB[testplayer]PW[GNU Go:3.8]RE[W+3.5]SZ[9];B[df]" + ";C[preprogrammed move C3]W[cg])") + + fx.check_command("boardsize", ['19'], "") + fx.check_command("fixed_handicap", ['3'], "D4 Q16 D16") + fx.check_command("komi", ['5.5'], "") + fx.check_command("play", ['W', 'A2'], "") + fx.check_command('gomill-savesgf', ['out3.sgf'], "") + tc.assertEqual( + scrub_sgf(fx.gtp_state._load_file('out3.sgf')).replace("\n", ""), + "(;FF[4]AB[dd][dp][pd]AP[gomill:VER]CA[UTF-8]DT[***]" + "GM[1]HA[3]KM[5.5]SZ[19];W[ar])") + fx.check_command( + 'gomill-savesgf', ['force_fail'], + "error writing file: [Errno 2] " + "No such file or directory: '/nonexistent_directory/foo.sgf'", + expect_failure=True) + +def test_get_last_move(tc): + fx = Gtp_state_fixture(tc) + fx.player.set_next_move("A3", "preprogrammed move A3") + fx.check_command('genmove', ['B'], "A3") + history_moves = fx.player.last_game_state.move_history + tc.assertEqual(gtp_states.get_last_move(history_moves, 'b'), (False, None)) + tc.assertEqual(gtp_states.get_last_move(history_moves, 'w'), (False, None)) + + fx.player.set_next_move("B3", "preprogrammed move B3") + fx.check_command('genmove', ['W'], "B3") + history_moves = fx.player.last_game_state.move_history + tc.assertEqual(gtp_states.get_last_move(history_moves, 'b'), (False, None)) + move_is_available, move = gtp_states.get_last_move(history_moves, 'w') + tc.assertIs(move_is_available, True) + tc.assertEqual(format_vertex(move), "A3") + + fx.check_command('genmove', ['B'], "pass") + history_moves = fx.player.last_game_state.move_history + move_is_available, move = gtp_states.get_last_move(history_moves, 'b') + tc.assertIs(move_is_available, True) + tc.assertEqual(format_vertex(move), "B3") + tc.assertEqual(gtp_states.get_last_move(history_moves, 'w'), (False, None)) + +def test_get_last_move_and_cookie(tc): + fx = Gtp_state_fixture(tc) + fx.player.set_next_move("A3", "preprogrammed move A3", "COOKIE 1") + fx.check_command('genmove', ['B'], "A3") + history_moves = fx.player.last_game_state.move_history + tc.assertEqual(gtp_states.get_last_move_and_cookie(history_moves, 'b'), + (False, None, None)) + tc.assertEqual(gtp_states.get_last_move_and_cookie(history_moves, 'w'), + (False, None, None)) + + fx.check_command('play', ['W', 'B3'], "") + fx.check_command('genmove', ['B'], "pass") + history_moves = fx.player.last_game_state.move_history + move_is_available, move, cookie = gtp_states.get_last_move_and_cookie( + history_moves, 'b') + tc.assertIs(move_is_available, True) + tc.assertEqual(format_vertex(move), "B3") + tc.assertIs(cookie, "COOKIE 1") + tc.assertEqual(gtp_states.get_last_move_and_cookie(history_moves, 'w'), + (False, None, None)) + diff --git a/gomill/gomill_tests/mcts_tuner_tests.py b/gomill/gomill_tests/mcts_tuner_tests.py new file mode 100644 index 0000000..a65f6b8 --- /dev/null +++ b/gomill/gomill_tests/mcts_tuner_tests.py @@ -0,0 +1,481 @@ +"""Tests for mcts_tuners.py""" + +from __future__ import with_statement, division + +from math import sqrt +import random +from textwrap import dedent +import cPickle as pickle + +from gomill import mcts_tuners +from gomill.game_jobs import Game_job, Game_job_result +from gomill.gtp_games import Game_result +from gomill.mcts_tuners import Parameter_config +from gomill.competitions import ( + Player_config, CompetitionError, ControlFileError) + +from gomill_tests import gomill_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def simple_make_candidate(*args): + if -1 in args: + raise ValueError("oops") + return Player_config("cand " + " ".join(map(str, args))) + +def default_config(): + return { + 'board_size' : 13, + 'komi' : 7.5, + 'players' : { + 'opp' : Player_config("test"), + }, + 'candidate_colour' : 'w', + 'opponent' : 'opp', + 'exploration_coefficient' : 0.2, + 'initial_visits' : 10, + 'initial_wins' : 5, + 'parameters' : [ + Parameter_config( + 'resign_at', + scale = float, + split = 12, + format = "rsn@ %.2f"), + + Parameter_config( + 'initial_wins', + scale = mcts_tuners.LINEAR(0, 100), + split = 10, + format = "iwins %d"), + ], + 'make_candidate' : simple_make_candidate, + } + +def test_bad_komi(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['komi'] = 6 + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertEqual(str(ar.exception), + "komi: must be fractional to prevent jigos") + +def test_parameter_config(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + comp.initialise_from_control_file(config) + tc.assertEqual(comp.tree.max_depth, 1) + tc.assertEqual(comp.format_engine_parameters((0.5, 23)), + "rsn@ 0.50; iwins 23") + tc.assertEqual(comp.format_engine_parameters(('x', 23)), + "[resign_at?x]; iwins 23") + tc.assertEqual(comp.format_optimiser_parameters((0.5, 0.23)), + "rsn@ 0.50; iwins 23") + tc.assertEqual(comp.scale_parameters((0.5, 0.23)), (0.5, 23)) + with tc.assertRaises(CompetitionError) as ar: + comp.scale_parameters((0.5, None)) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + error from scale for initial_wins + TypeError: unsupported operand type(s) for *: 'NoneType' and 'float' + traceback (most recent call last): + mcts_tuners|__call__ + failing line: + result = (f * self.range) + self.lower_bound + """)) + + tc.assertRaisesRegexp( + ValueError, "'code' not specified", + comp.parameter_spec_from_config, Parameter_config()) + tc.assertRaisesRegexp( + ValueError, "code specified as both positional and keyword argument", + comp.parameter_spec_from_config, + Parameter_config('pa1', code='pa2', scale=float, split=2, format="%s")) + tc.assertRaisesRegexp( + ValueError, "too many positional arguments", + comp.parameter_spec_from_config, + Parameter_config('pa1', float, scale=float, split=2, format="%s")) + tc.assertRaisesRegexp( + ValueError, "'scale': invalid callable", + comp.parameter_spec_from_config, + Parameter_config('pa1', scale=None, split=2, format="%s")) + pspec = comp.parameter_spec_from_config( + Parameter_config('pa1', scale=float, split=2)) + tc.assertRaisesRegexp( + ControlFileError, "'format': invalid format string", + comp.parameter_spec_from_config, + Parameter_config('pa1', scale=float, split=2, format="nopct")) + +def test_bad_parameter_config(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'].append( + Parameter_config( + 'bad', + scale = mcts_tuners.LOG(0.0, 1.0), + split = 10)) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + parameter bad: 'scale': invalid parameters for LOG: + lower bound is zero""")) + +def test_nonsense_parameter_config(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'].append(99) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + 'parameters': item 2: not a Parameter""")) + +def test_nocode_parameter_config(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'].append(Parameter_config()) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + parameter 2: 'code' not specified""")) + +def test_scale_check(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'].append( + Parameter_config( + 'bad', + scale = str.split, + split = 10)) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + parameter bad: error from scale (applied to 0.05) + TypeError: split-wants-float-not-str + traceback (most recent call last): + """), fixups=[ + ("descriptor 'split' requires a 'str' object but received a 'float'", + "split-wants-float-not-str"), + ("unbound method split() must be called with str instance as " + "first argument (got float instance instead)", + "split-wants-float-not-str"), + ]) + +def test_format_validation(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'].append( + Parameter_config( + 'bad', + scale = str, + split = 10, + format = "bad: %.2f")) + tc.assertRaisesRegexp( + ControlFileError, "'format': invalid format string", + comp.initialise_from_control_file, config) + +def test_make_candidate(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + comp.initialise_from_control_file(config) + cand = comp.make_candidate('c#1', (0.5, 23)) + tc.assertEqual(cand.code, 'c#1') + tc.assertListEqual(cand.cmd_args, ['cand', '0.5', '23']) + with tc.assertRaises(CompetitionError) as ar: + comp.make_candidate('c#1', (-1, 23)) + tc.assertTracebackStringEqual(str(ar.exception), dedent("""\ + error from make_candidate() + ValueError: oops + traceback (most recent call last): + mcts_tuner_tests|simple_make_candidate + failing line: + raise ValueError("oops") + """)) + +def test_get_player_checks(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + comp.initialise_from_control_file(config) + checks = comp.get_player_checks() + tc.assertEqual(len(checks), 2) + tc.assertEqual(checks[0].player.code, "candidate") + tc.assertEqual(checks[0].player.cmd_args, ['cand', str(1/24), '5.0']) + tc.assertEqual(checks[1].player.code, "opp") + tc.assertEqual(checks[1].player.cmd_args, ['test']) + +def test_linear_scale(tc): + lsf = mcts_tuners.Linear_scale_fn(20.0, 30.0) + tc.assertEqual(lsf(0.0), 20.0) + tc.assertEqual(lsf(1.0), 30.0) + tc.assertEqual(lsf(0.5), 25.0) + tc.assertEqual(lsf(0.49), 24.9) + + lsi = mcts_tuners.Linear_scale_fn(20.0, 30.0, integer=True) + tc.assertEqual(lsi(0.0), 20) + tc.assertEqual(lsi(1.0), 30) + tc.assertEqual(lsi(0.49), 25) + tc.assertEqual(lsi(0.51), 25) + +def test_log_scale(tc): + lsf = mcts_tuners.Log_scale_fn(2, 200000) + tc.assertAlmostEqual(lsf(0.0), 2.0) + tc.assertAlmostEqual(lsf(0.2), 20.0) + tc.assertAlmostEqual(lsf(0.4), 200.0) + tc.assertAlmostEqual(lsf(0.5), 2*sqrt(100000.00)) + tc.assertAlmostEqual(lsf(0.6), 2000.0) + tc.assertAlmostEqual(lsf(0.8), 20000.0) + tc.assertAlmostEqual(lsf(1.0), 200000.0) + + lsi = mcts_tuners.Log_scale_fn(1, 100, integer=True) + tc.assertAlmostEqual(lsi(0.1), 2) + + lsn = mcts_tuners.Log_scale_fn(-2, -200) + tc.assertAlmostEqual(lsn(0.5), -20) + + tc.assertRaises(ValueError, mcts_tuners.Log_scale_fn, 1, -2) + + +def test_explicit_scale(tc): + tc.assertRaises(ValueError, mcts_tuners.Explicit_scale_fn, []) + + pvalues = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] + + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'] = [ + Parameter_config( + 'range', + scale = mcts_tuners.EXPLICIT(pvalues), + split = len(pvalues))] + comp.initialise_from_control_file(config) + candidate_sees = [ + comp.scale_parameters(comp.tree.parameters_for_path([i]))[0] + for i, _ in enumerate(pvalues) + ] + tc.assertEqual(candidate_sees, pvalues) + +def test_integer_scale_example(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + config = default_config() + config['parameters'] = [ + Parameter_config( + 'range', + scale = mcts_tuners.LINEAR(-.5, 10.5, integer=True), + split = 11)] + comp.initialise_from_control_file(config) + candidate_sees = [ + comp.scale_parameters(comp.tree.parameters_for_path([i]))[0] + for i in xrange(11) + ] + tc.assertEqual(candidate_sees, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + + +def test_tree(tc): + tree1 = mcts_tuners.Tree( + splits=[3, 3], + max_depth=5, + exploration_coefficient=0.5, + initial_visits=10, + initial_wins=5, + parameter_formatter=str, + ) + + tree2 = mcts_tuners.Tree( + splits=[2, 4], + max_depth=5, + exploration_coefficient=0.5, + initial_visits=10, + initial_wins=5, + parameter_formatter=str, + ) + + tc.assertEqual(tree1.max_depth, 5) + + tc.assertEqual(tree1._cube_coordinates, [ + (0, 0), (0, 1), (0, 2), + (1, 0), (1, 1), (1, 2), + (2, 0), (2, 1), (2, 2), + ]) + + tc.assertEqual(tree2._cube_coordinates, [ + (0, 0), (0, 1), (0, 2), (0, 3), + (1, 0), (1, 1), (1, 2), (1, 3), + ]) + + def scaleup(vector, scales): + """Multiply each member of 'vector' by the corresponding scale. + + Rounds to nearest integer if the difference is very small. + + """ + result = [] + for v, scale in zip(vector, scales): + f = v*scale + i = int(f+.5) + if abs(f - i) > 0.0000000001: + result.append(f) + else: + result.append(i) + return result + + def pfp1(choice_path): + optimiser_params = tree1.parameters_for_path(choice_path) + scale = 3**len(choice_path) * 2 + return scaleup(optimiser_params, [scale, scale]) + + # scale is 1/6 + tc.assertEqual(pfp1([0]), [1, 1]) + tc.assertEqual(pfp1([1]), [1, 3]) + tc.assertEqual(pfp1([3]), [3, 1]) + tc.assertEqual(pfp1([8]), [5, 5]) + # scale is 1/18 + tc.assertEqual(pfp1([0, 0]), [1, 1]) + tc.assertEqual(pfp1([0, 1]), [1, 3]) + tc.assertEqual(pfp1([3, 1]), [7, 3]) + tc.assertEqual(pfp1([3, 4]), [9, 3]) + + def pfp2(choice_path): + optimiser_params = tree2.parameters_for_path(choice_path) + scale1 = 2**len(choice_path) * 2 + scale2 = 4**len(choice_path) * 2 + return scaleup(optimiser_params, [scale1, scale2]) + + # scale is 1/4, 1/8 + tc.assertEqual(pfp2([0]), [1, 1]) + tc.assertEqual(pfp2([1]), [1, 3]) + tc.assertEqual(pfp2([2]), [1, 5]) + tc.assertEqual(pfp2([3]), [1, 7]) + tc.assertEqual(pfp2([4]), [3, 1]) + tc.assertEqual(pfp2([5]), [3, 3]) + # scale is 1/8, 1/32 + tc.assertEqual(pfp2([0, 0]), [1, 1]) + tc.assertEqual(pfp2([0, 1]), [1, 3]) + tc.assertEqual(pfp2([0, 2]), [1, 5]) + tc.assertEqual(pfp2([0, 3]), [1, 7]) + tc.assertEqual(pfp2([0, 4]), [3, 1]) + tc.assertEqual(pfp2([1, 0]), [1, 9]) + tc.assertEqual(pfp2([7, 7]), [7, 31]) + + +def test_play(tc): + comp = mcts_tuners.Mcts_tuner('mctstest') + comp.initialise_from_control_file(default_config()) + comp.set_clean_status() + tree = comp.tree + tc.assertEqual(comp.outstanding_simulations, {}) + + tc.assertEqual(tree.root.visits, 10) + tc.assertEqual(tree.root.wins, 5) + tc.assertEqual(sum(node.visits-10 for node in tree.root.children), 0) + tc.assertEqual(sum(node.wins-5 for node in tree.root.children), 0) + + job1 = comp.get_game() + tc.assertIsInstance(job1, Game_job) + tc.assertEqual(job1.game_id, '0') + tc.assertEqual(job1.player_b.code, 'opp') + tc.assertEqual(job1.player_w.code, '#0') + tc.assertEqual(job1.board_size, 13) + tc.assertEqual(job1.komi, 7.5) + tc.assertEqual(job1.move_limit, 1000) + tc.assertIs(job1.use_internal_scorer, False) + tc.assertEqual(job1.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(job1.game_data, 0) + tc.assertEqual(job1.sgf_event, 'mctstest') + tc.assertRegexpMatches(job1.sgf_note, '^Candidate parameters: rsn@ ') + tc.assertItemsEqual(comp.outstanding_simulations.keys(), [0]) + + job2 = comp.get_game() + tc.assertIsInstance(job2, Game_job) + tc.assertEqual(job2.game_id, '1') + tc.assertEqual(job2.player_b.code, 'opp') + tc.assertEqual(job2.player_w.code, '#1') + tc.assertItemsEqual(comp.outstanding_simulations.keys(), [0, 1]) + + result1 = Game_result({'b' : 'opp', 'w' : '#1'}, 'w') + result1.sgf_result = "W+8.5" + response1 = Game_job_result() + response1.game_id = job1.game_id + response1.game_result = result1 + response1.engine_names = { + 'opp' : 'opp engine:v1.2.3', + '#0' : 'candidate engine', + } + response1.engine_descriptions = { + 'opp' : 'opp engine:v1.2.3', + '#0' : 'candidate engine description', + } + response1.game_data = job1.game_data + comp.process_game_result(response1) + tc.assertItemsEqual(comp.outstanding_simulations.keys(), [1]) + + tc.assertEqual(tree.root.visits, 11) + tc.assertEqual(tree.root.wins, 6) + tc.assertEqual(sum(node.visits-10 for node in tree.root.children), 1) + tc.assertEqual(sum(node.wins-5 for node in tree.root.children), 1) + + comp2 = mcts_tuners.Mcts_tuner('mctstest') + comp2.initialise_from_control_file(default_config()) + status = pickle.loads(pickle.dumps(comp.get_status())) + comp2.set_status(status) + tc.assertEqual(comp2.tree.root.visits, 11) + tc.assertEqual(comp2.tree.root.wins, 6) + tc.assertEqual(sum(node.visits-10 for node in comp2.tree.root.children), 1) + tc.assertEqual(sum(node.wins-5 for node in comp2.tree.root.children), 1) + + config3 = default_config() + # changed split + config3['parameters'][0] = Parameter_config( + 'resign_at', + scale = float, + split = 11, + format = "rsn@ %.2f") + comp3 = mcts_tuners.Mcts_tuner('mctstest') + comp3.initialise_from_control_file(config3) + status = pickle.loads(pickle.dumps(comp.get_status())) + with tc.assertRaises(CompetitionError) as ar: + comp3.set_status(status) + tc.assertEqual(str(ar.exception), + "status file is inconsistent with control file") + + config4 = default_config() + # changed upper bound + config4['parameters'][1] = Parameter_config( + 'initial_wins', + scale = mcts_tuners.LINEAR(0, 200), + split = 10, + format = "iwins %d") + comp4 = mcts_tuners.Mcts_tuner('mctstest') + comp4.initialise_from_control_file(config4) + status = pickle.loads(pickle.dumps(comp.get_status())) + with tc.assertRaises(CompetitionError) as ar: + comp4.set_status(status) + tc.assertEqual(str(ar.exception), + "status file is inconsistent with control file") + + +def _disabled_test_tree_run(tc): + # Something like this test can be useful when changing the tree code, + # if you want to verify that you're not changing behaviour. + + tree = mcts_tuners.Tree( + splits=[2, 3], + max_depth=5, + exploration_coefficient=0.5, + initial_visits=10, + initial_wins=5, + parameter_formatter=str, + ) + + tree.new_root() + random.seed(12345) + for i in range(1100): + simulation = mcts_tuners.Simulation(tree) + simulation.run() + simulation.update_stats(candidate_won=random.randrange(2)) + tc.assertEqual(simulation.get_parameters(), + [0.0625, 0.42592592592592593]) + tc.assertEqual(tree.node_count, 1597) + tc.assertEqual(simulation.choice_path, [1, 0, 2]) + tc.assertEqual(tree.retrieve_best_parameters(), + [0.609375, 0.68930041152263366]) + diff --git a/gomill/gomill_tests/playoff_tests.py b/gomill/gomill_tests/playoff_tests.py new file mode 100644 index 0000000..4a253cf --- /dev/null +++ b/gomill/gomill_tests/playoff_tests.py @@ -0,0 +1,570 @@ +"""Tests for playoffs.py""" + +from __future__ import with_statement + +from textwrap import dedent +import cPickle as pickle + +from gomill import competitions +from gomill import playoffs +from gomill.gtp_games import Game_result +from gomill.game_jobs import Game_job, Game_job_result +from gomill.competitions import ( + Player_config, NoGameAvailable, CompetitionError, ControlFileError) +from gomill.playoffs import Matchup_config + +from gomill_tests import competition_test_support +from gomill_tests import gomill_test_support +from gomill_tests import test_framework +from gomill_tests.competition_test_support import ( + fake_response, check_screen_report) + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def check_short_report(tc, comp, expected_matchups, expected_players, + competition_name="testcomp"): + """Check that a playoff's short report is as expected.""" + expected = ("playoff: %s\n\n%s\n%s\n" % + (competition_name, expected_matchups, expected_players)) + tc.assertMultiLineEqual(competition_test_support.get_short_report(comp), + expected) + +expected_fake_players = dedent("""\ + player t1: t1 engine + testdescription + player t2: t2 engine:v1.2.3 + """) + + +class Playoff_fixture(test_framework.Fixture): + """Fixture setting up a Playoff. + + attributes: + comp -- Playoff + + """ + def __init__(self, tc, config=None): + if config is None: + config = default_config() + self.tc = tc + self.comp = playoffs.Playoff('testcomp') + self.comp.initialise_from_control_file(config) + self.comp.set_clean_status() + + def check_screen_report(self, expected): + """Check that the screen report is as expected.""" + check_screen_report(self.tc, self.comp, expected) + + def check_short_report(self, *args, **kwargs): + """Check that the short report is as expected.""" + check_short_report(self.tc, self.comp, *args, **kwargs) + + +def default_config(): + return { + 'players' : { + 't1' : Player_config("test1"), + 't2' : Player_config("test2"), + }, + 'board_size' : 13, + 'komi' : 7.5, + 'matchups' : [ + Matchup_config('t1', 't2', alternating=True), + ], + } + + +def test_basic_config(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['description'] = "default\nconfig" + config['matchups'] = [ + Matchup_config( + 't1', 't2', board_size=9, komi=0.5, alternating=True, + handicap=6, handicap_style='free', + move_limit=50, + scorer="internal", internal_scorer_handicap_compensation='no', + number_of_games=20), + Matchup_config('t2', 't1', id='m1'), + Matchup_config('t1', 't2'), + ] + comp.initialise_from_control_file(config) + tc.assertEqual(comp.description, "default\nconfig") + + comp.set_clean_status() + tr = comp.get_tournament_results() + m0 = tr.get_matchup('0') + m1 = tr.get_matchup('m1') + m2 = tr.get_matchup('2') + + tc.assertListEqual(tr.get_matchup_ids(), ['0', 'm1', '2']) + tc.assertDictEqual(tr.get_matchups(), {'0' : m0, 'm1' : m1, '2' : m2}) + + tc.assertEqual(m0.player_1, 't1') + tc.assertEqual(m0.player_2, 't2') + tc.assertEqual(m0.board_size, 9) + tc.assertEqual(m0.komi, 0.5) + tc.assertIs(m0.alternating, True) + tc.assertEqual(m0.handicap, 6) + tc.assertEqual(m0.handicap_style, 'free') + tc.assertEqual(m0.move_limit, 50) + tc.assertEqual(m0.scorer, 'internal') + tc.assertEqual(m0.internal_scorer_handicap_compensation, 'no') + tc.assertEqual(m0.number_of_games, 20) + + tc.assertEqual(m1.player_1, 't2') + tc.assertEqual(m1.player_2, 't1') + tc.assertEqual(m1.board_size, 13) + tc.assertEqual(m1.komi, 7.5) + tc.assertIs(m1.alternating, False) + tc.assertEqual(m1.handicap, None) + tc.assertEqual(m1.handicap_style, 'fixed') + tc.assertEqual(m1.move_limit, 1000) + tc.assertEqual(m1.scorer, 'players') + tc.assertEqual(m1.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(m1.number_of_games, None) + +def test_nonsense_matchup_config(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(99) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + 'matchups': item 1: not a Matchup""")) + +def test_bad_matchup_config(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(Matchup_config()) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 1: not enough arguments""")) + +def test_bad_matchup_config_bad_id(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(Matchup_config('t1', 't2', id=99)) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 1: 'id': not a string""")) + +def test_bad_matchup_config_bad_setting(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(Matchup_config('t1', 't2', board_size="X")) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 1: 'board_size': invalid integer""")) + +def test_bad_matchup_config_unknown_player(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(Matchup_config('t1', 'nonex')) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 1: unknown player nonex""")) + +def test_bad_matchup_config_no_board_size(tc): + comp = playoffs.Playoff('test') + config = default_config() + del config['board_size'] + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 0: 'board_size' not specified""")) + +def test_bad_matchup_config_bad_handicap(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'].append(Matchup_config('t1', 't2', handicap=10)) + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertMultiLineEqual(str(ar.exception), dedent("""\ + matchup 1: fixed handicap out of range for board size 13""")) + +def test_matchup_config_board_size_in_matchup_only(tc): + comp = playoffs.Playoff('test') + config = default_config() + del config['board_size'] + config['matchups'] = [Matchup_config('t1', 't2', alternating=True, + board_size=9)] + comp.initialise_from_control_file(config) + comp.set_clean_status() + tr = comp.get_tournament_results() + m0 = tr.get_matchup('0') + tc.assertEqual(m0.board_size, 9) + +def test_matchup_name(tc): + comp = playoffs.Playoff('test') + config = default_config() + config['matchups'] = [Matchup_config('t1', 't2', name="asd")] + comp.initialise_from_control_file(config) + comp.set_clean_status() + tr = comp.get_tournament_results() + m0 = tr.get_matchup('0') + tc.assertEqual(m0.name, "asd") + +def test_global_handicap_validation(tc): + comp = playoffs.Playoff('testcomp') + config = default_config() + config['board_size'] = 12 + config['handicap'] = 6 + with tc.assertRaises(ControlFileError) as ar: + comp.initialise_from_control_file(config) + tc.assertEqual(str(ar.exception), + "default fixed handicap out of range for board size 12") + + +def test_game_id_format(tc): + config = default_config() + config['matchups'][0] = Matchup_config('t1', 't2', number_of_games=1000) + fx = Playoff_fixture(tc, config) + tc.assertEqual(fx.comp.get_game().game_id, '0_000') + +def test_get_player_checks(tc): + comp = playoffs.Playoff('testcomp') + config = default_config() + config['players']['t3'] = Player_config("test3") + config['matchups'].append( + Matchup_config('t1', 't3', number_of_games=0), + ), + comp.initialise_from_control_file(config) + checks = comp.get_player_checks() + tc.assertEqual(len(checks), 2) + tc.assertEqual(checks[0].board_size, 13) + tc.assertEqual(checks[0].komi, 7.5) + tc.assertEqual(checks[0].player.code, "t1") + tc.assertEqual(checks[0].player.cmd_args, ['test1']) + tc.assertEqual(checks[1].player.code, "t2") + tc.assertEqual(checks[1].player.cmd_args, ['test2']) + +def test_play(tc): + fx = Playoff_fixture(tc) + tc.assertIsNone(fx.comp.description) + + job1 = fx.comp.get_game() + tc.assertIsInstance(job1, Game_job) + tc.assertEqual(job1.game_id, '0_0') + tc.assertEqual(job1.player_b.code, 't1') + tc.assertEqual(job1.player_w.code, 't2') + tc.assertEqual(job1.board_size, 13) + tc.assertEqual(job1.komi, 7.5) + tc.assertEqual(job1.move_limit, 1000) + tc.assertIs(job1.use_internal_scorer, False) + tc.assertEqual(job1.internal_scorer_handicap_compensation, 'full') + tc.assertEqual(job1.game_data, ('0', 0)) + tc.assertIsNone(job1.sgf_filename) + tc.assertIsNone(job1.sgf_dirname) + tc.assertIsNone(job1.void_sgf_dirname) + tc.assertEqual(job1.sgf_event, 'testcomp') + tc.assertIsNone(job1.gtp_log_pathname) + + job2 = fx.comp.get_game() + tc.assertIsInstance(job2, Game_job) + tc.assertEqual(job2.game_id, '0_1') + tc.assertEqual(job2.player_b.code, 't2') + tc.assertEqual(job2.player_w.code, 't1') + + result1 = Game_result({'b' : 't1', 'w' : 't2'}, 'b') + result1.sgf_result = "B+8.5" + response1 = Game_job_result() + response1.game_id = job1.game_id + response1.game_result = result1 + response1.engine_names = { + 't1' : 't1 engine:v1.2.3', + 't2' : 't2 engine', + } + response1.engine_descriptions = { + 't1' : 't1 engine:v1.2.3', + 't2' : 't2 engine\ntest \xc2\xa3description', + } + response1.game_data = job1.game_data + fx.comp.process_game_result(response1) + + expected_report = dedent("""\ + t1 v t2 (1 games) + board size: 13 komi: 7.5 + wins + t1 1 100.00% (black) + t2 0 0.00% (white) + """) + expected_players = dedent("""\ + player t1: t1 engine:v1.2.3 + player t2: t2 engine + test \xc2\xa3description + """) + fx.check_screen_report(expected_report) + fx.check_short_report(expected_report, expected_players) + + tc.assertListEqual( + fx.comp.get_tournament_results().get_matchup_results('0'), [result1]) + +def test_play_many(tc): + fx = Playoff_fixture(tc) + + jobs = [fx.comp.get_game() for _ in range(8)] + for i in [0, 3]: + response = fake_response(jobs[i], 'b') + fx.comp.process_game_result(response) + jobs += [fx.comp.get_game() for _ in range(3)] + for i in [4, 2, 6, 7]: + response = fake_response(jobs[i], 'w') + fx.comp.process_game_result(response) + + fx.check_screen_report(dedent("""\ + t1 v t2 (6 games) + board size: 13 komi: 7.5 + wins black white + t1 2 33.33% 1 25.00% 1 50.00% + t2 4 66.67% 1 50.00% 3 75.00% + 2 33.33% 4 66.67% + """)) + + tc.assertEqual( + len(fx.comp.get_tournament_results().get_matchup_results('0')), 6) + + #tc.assertEqual(fx.comp.scheduler.allocators['0'].issued, 11) + #tc.assertEqual(fx.comp.scheduler.allocators['0'].fixed, 6) + + comp2 = competition_test_support.check_round_trip( + tc, fx.comp, default_config()) + + #tc.assertEqual(comp2.scheduler.allocators['0'].issued, 6) + #tc.assertEqual(comp2.scheduler.allocators['0'].fixed, 6) + + jobs2 = [comp2.get_game() for _ in range(4)] + tc.assertListEqual([job.game_id for job in jobs2], + ['0_1', '0_5', '0_8', '0_9']) + tr = comp2.get_tournament_results() + tc.assertEqual(len(tr.get_matchup_results('0')), 6) + ms = tr.get_matchup_stats('0') + tc.assertEqual(ms.total, 6) + tc.assertEqual(ms.wins_1, 2) + tc.assertEqual(ms.wins_b, 2) + +def test_jigo_reporting(tc): + fx = Playoff_fixture(tc) + + def winner(i): + if i in (0, 3): + return 'b' + elif i in (2, 4): + return 'w' + else: + return None + jobs = [fx.comp.get_game() for _ in range(8)] + for i in range(6): + response = fake_response(jobs[i], winner(i)) + fx.comp.process_game_result(response) + fx.check_screen_report(dedent("""\ + t1 v t2 (6 games) + board size: 13 komi: 7.5 + wins black white + t1 2 33.33% 2 66.67% 1 33.33% + t2 4 66.67% 2 66.67% 3 100.00% + 3 50.00% 3 50.00% + """)) + + response = fake_response(jobs[6], None) + fx.comp.process_game_result(response) + fx.check_screen_report(dedent("""\ + t1 v t2 (7 games) + board size: 13 komi: 7.5 + wins black white + t1 2.5 35.71% 2.5 62.50% 1.5 50.00% + t2 4.5 64.29% 2.5 83.33% 3.5 87.50% + 3.5 50.00% 3.5 50.00% + """)) + +def test_unknown_result_reporting(tc): + fx = Playoff_fixture(tc) + + def winner(i): + if i in (0, 3): + return 'b' + elif i in (2, 4): + return 'w' + else: + return 'unknown' + jobs = [fx.comp.get_game() for _ in range(8)] + for i in range(6): + response = fake_response(jobs[i], winner(i)) + fx.comp.process_game_result(response) + fx.check_screen_report(dedent("""\ + t1 v t2 (6 games) + unknown results: 2 33.33% + board size: 13 komi: 7.5 + wins black white + t1 1 16.67% 1 33.33% 0 0.00% + t2 3 50.00% 1 33.33% 2 66.67% + 2 33.33% 2 33.33% + """)) + +def test_self_play(tc): + config = default_config() + config['matchups'] = [ + Matchup_config('t1', 't1', alternating=True), + Matchup_config('t1', 't1', alternating=False), + ] + fx = Playoff_fixture(tc, config) + + jobs = [fx.comp.get_game() for _ in range(20)] + for i, job in enumerate(jobs): + response = fake_response(job, 'b' if i < 9 else 'w') + fx.comp.process_game_result(response) + fx.check_screen_report(dedent("""\ + t1 v t1#2 (10 games) + board size: 13 komi: 7.5 + wins black white + t1 6 60.00% 3 60.00% 3 60.00% + t1#2 4 40.00% 2 40.00% 2 40.00% + 5 50.00% 5 50.00% + + t1 v t1#2 (10 games) + board size: 13 komi: 7.5 + wins + t1 4 40.00% (black) + t1#2 6 60.00% (white) + """)) + + competition_test_support.check_round_trip(tc, fx.comp, config) + + +def test_bad_state(tc): + fx = Playoff_fixture(tc) + bad_status = fx.comp.get_status() + del bad_status['scheduler'] + + comp2 = playoffs.Playoff('testcomp') + comp2.initialise_from_control_file(default_config()) + tc.assertRaises(KeyError, comp2.set_status, bad_status) + +def test_matchup_change(tc): + fx = Playoff_fixture(tc) + + jobs = [fx.comp.get_game() for _ in range(8)] + for i in [0, 2, 3, 4, 6, 7]: + response = fake_response(jobs[i], ('b' if i in (0, 3) else 'w')) + fx.comp.process_game_result(response) + + fx.check_screen_report(dedent("""\ + t1 v t2 (6 games) + board size: 13 komi: 7.5 + wins black white + t1 2 33.33% 1 25.00% 1 50.00% + t2 4 66.67% 1 50.00% 3 75.00% + 2 33.33% 4 66.67% + """)) + + config2 = default_config() + config2['players']['t3'] = Player_config("test3") + config2['matchups'][0] = Matchup_config('t1', 't3', alternating=True) + comp2 = playoffs.Playoff('testcomp') + comp2.initialise_from_control_file(config2) + + status = pickle.loads(pickle.dumps(fx.comp.get_status())) + with tc.assertRaises(CompetitionError) as ar: + comp2.set_status(status) + tc.assertEqual( + str(ar.exception), + "existing results for matchup 0 are inconsistent with control file:\n" + "result players are t1,t2;\n" + "control file players are t1,t3") + +def test_matchup_reappearance(tc): + # Test that if a matchup is removed and added again, we remember the game + # number. Test that we report the 'ghost' matchup in the short report (but + # not the screen report). + config1 = default_config() + config1['matchups'].append(Matchup_config('t2', 't1')) + config2 = default_config() + config3 = default_config() + config3['matchups'].append(Matchup_config('t2', 't1')) + + comp1 = playoffs.Playoff('testcomp') + comp1.initialise_from_control_file(config1) + comp1.set_clean_status() + jobs1 = [comp1.get_game() for _ in range(8)] + for job in jobs1: + comp1.process_game_result(fake_response(job, 'b')) + tc.assertListEqual( + [job.game_id for job in jobs1], + ['0_0', '1_0', '0_1', '1_1', '0_2', '1_2', '0_3', '1_3']) + expected_matchups_1 = dedent("""\ + t1 v t2 (4 games) + board size: 13 komi: 7.5 + wins black white + t1 2 50.00% 2 100.00% 0 0.00% + t2 2 50.00% 2 100.00% 0 0.00% + 4 100.00% 0 0.00% + + t2 v t1 (4 games) + board size: 13 komi: 7.5 + wins + t2 4 100.00% (black) + t1 0 0.00% (white) + """) + check_screen_report(tc, comp1, expected_matchups_1) + check_short_report(tc, comp1, expected_matchups_1, expected_fake_players) + + comp2 = playoffs.Playoff('testcomp') + comp2.initialise_from_control_file(config2) + comp2.set_status(pickle.loads(pickle.dumps(comp1.get_status()))) + jobs2 = [comp2.get_game() for _ in range(4)] + tc.assertListEqual( + [job.game_id for job in jobs2], + ['0_4', '0_5', '0_6', '0_7']) + for job in jobs2: + comp2.process_game_result(fake_response(job, 'b')) + expected_matchups_2 = dedent("""\ + t1 v t2 (8 games) + board size: 13 komi: 7.5 + wins black white + t1 4 50.00% 4 100.00% 0 0.00% + t2 4 50.00% 4 100.00% 0 0.00% + 8 100.00% 0 0.00% + """) + check_screen_report(tc, comp2, expected_matchups_2) + expected_matchups_2b = dedent("""\ + t2 v t1 (4 games) + ?? (missing from control file) + wins + t2 4 100.00% (black) + t1 0 0.00% (white) + """) + check_short_report( + tc, comp2, + expected_matchups_2 + "\n" + expected_matchups_2b, + expected_fake_players) + + comp3 = playoffs.Playoff('testcomp') + comp3.initialise_from_control_file(config3) + comp3.set_status(pickle.loads(pickle.dumps(comp2.get_status()))) + jobs3 = [comp3.get_game() for _ in range(8)] + tc.assertListEqual( + [job.game_id for job in jobs3], + ['1_4', '1_5', '1_6', '1_7', '0_8', '1_8', '0_9', '1_9']) + expected_matchups_3 = dedent("""\ + t1 v t2 (8 games) + board size: 13 komi: 7.5 + wins black white + t1 4 50.00% 4 100.00% 0 0.00% + t2 4 50.00% 4 100.00% 0 0.00% + 8 100.00% 0 0.00% + + t2 v t1 (4 games) + board size: 13 komi: 7.5 + wins + t2 4 100.00% (black) + t1 0 0.00% (white) + """) + check_screen_report(tc, comp3, expected_matchups_3) + check_short_report(tc, comp3, expected_matchups_3, expected_fake_players) + diff --git a/gomill/gomill_tests/ringmaster_test_support.py b/gomill/gomill_tests/ringmaster_test_support.py new file mode 100644 index 0000000..3443275 --- /dev/null +++ b/gomill/gomill_tests/ringmaster_test_support.py @@ -0,0 +1,93 @@ +"""Test support code for testing Ringmasters.""" + +from collections import defaultdict +from cStringIO import StringIO + +from gomill import ringmasters +from gomill import ringmaster_presenters + +class Test_presenter(ringmaster_presenters.Presenter): + """Presenter which stores the messages.""" + def __init__(self): + ringmaster_presenters.Presenter.__init__(self) + self.channels = defaultdict(list) + + shows_warnings_only = False + + def clear(self, channel): + self.channels[channel] = [] + + def say(self, channel, s): + self.channels[channel].append(s) + + def refresh(self): + pass + + def recent_messages(self, channel): + """Retrieve messages sent since the channel was last cleared. + + Returns a list of strings. + + """ + return self.channels[channel][:] + + +class Testing_ringmaster(ringmasters.Ringmaster): + """Variant of ringmaster suitable for use in tests. + + This doesn't read from or write to the filesystem. + + (If you're testing run(), make sure record_games is False, and + discard_stderr is True for each player.) + + (Currently, write_status is made to do nothing, so it's not usefully + testable.) + + Instantiate with the control file contents as an 8-bit string. + + It will act as if the control file had been loaded from + /nonexistent/ctl/test.ctl. + + You'll want to run set_display_mode('test') with this. + + """ + def __init__(self, control_file_contents): + self._control_file_contents = control_file_contents + self._test_status = None + self._written_status = None + ringmasters.Ringmaster.__init__(self, '/nonexistent/ctl/test.ctl') + + _presenter_classes = { + 'test' : Test_presenter, + } + + def _open_files(self): + self.logfile = StringIO() + self.historyfile = StringIO() + + def _close_files(self): + # Don't want to close the StringIOs + pass + + def _read_control_file(self): + return self._control_file_contents + + def set_test_status(self, test_status): + """Specify the value that will be loaded from the state file. + + test_status -- fake state file contents + + test_status should be a pair (status_format_version, status dict) + + """ + self._test_status = test_status + + def _load_status(self): + return self._test_status + + def status_file_exists(self): + return (self._test_status is not None) + + def _write_status(self, value): + self._written_status = value + diff --git a/gomill/gomill_tests/ringmaster_tests.py b/gomill/gomill_tests/ringmaster_tests.py new file mode 100644 index 0000000..5c0247d --- /dev/null +++ b/gomill/gomill_tests/ringmaster_tests.py @@ -0,0 +1,520 @@ +"""Tests for ringmaster.py.""" + +import os +import re +from textwrap import dedent + +from gomill_tests import test_framework +from gomill_tests import gomill_test_support +from gomill_tests import ringmaster_test_support +from gomill_tests import gtp_engine_fixtures +from gomill_tests.playoff_tests import fake_response + +from gomill.ringmasters import RingmasterError + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +class Ringmaster_fixture(test_framework.Fixture): + """Fixture setting up a Ringmaster with mock suprocesses. + + Instantiate with testcase, the text to be used as the contents of the + control file, and a list of strings to be added (as a line each) to the end + of the control file. + + attributes: + ringmaster -- Testing_ringmaster + msf -- Mock_subprocess_fixture + + See Mock_subprocess_gtp_channel for an explanation of how player command + lines are interpreted. + + """ + def __init__(self, tc, control_file_contents, extra_lines=[]): + self.ringmaster = ringmaster_test_support.Testing_ringmaster( + control_file_contents + "\n".join(extra_lines)) + self.ringmaster.set_display_mode('test') + self.msf = gtp_engine_fixtures.Mock_subprocess_fixture(tc) + + def messages(self, channel): + """Return messages sent to the specified channel.""" + return self.ringmaster.presenter.recent_messages(channel) + + def initialise_clean(self): + """Initialise the ringmaster (with clean status).""" + self.ringmaster.set_clean_status() + self.ringmaster._open_files() + self.ringmaster._initialise_presenter() + self.ringmaster._initialise_terminal_reader() + + def initialise_with_state(self, ringmaster_status): + """Initialise the ringmaster with specified status.""" + self.ringmaster.set_test_status(ringmaster_status) + self.ringmaster.load_status() + self.ringmaster._open_files() + self.ringmaster._initialise_presenter() + self.ringmaster._initialise_terminal_reader() + + def get_job(self): + """Initialise the ringmaster, and call get_job() once.""" + self.initialise_clean() + return self.ringmaster.get_job() + + def get_log(self): + """Retrieve the log file contents with timestamps scrubbed out.""" + s = self.ringmaster.logfile.getvalue() + s = re.sub(r"(?<= at )([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})", + "***", s) + return s + + def get_history(self): + """Retrieve the history file contents.""" + return self.ringmaster.historyfile.getvalue() + + def get_written_state(self): + """Return the unpickled value written to the state file.""" + return self.ringmaster._written_status + + +playoff_ctl = """ + +competition_type = 'playoff' + +description = 'gomill_tests playoff.' + +players = { + 'p1' : Player('test'), + 'p2' : Player('test'), + } + +move_limit = 400 +record_games = False +board_size = 9 +komi = 7.5 +scorer = 'internal' + +number_of_games = 400 + +matchups = [ + Matchup('p1', 'p2'), + ] + +""" + +allplayall_ctl = """ + +competition_type = 'allplayall' + +description = 'gomill_tests allplayall_ctl.' + +players = { + 'p1' : Player('test'), + 'p2' : Player('test'), + } + +move_limit = 400 +record_games = False +board_size = 9 +komi = 7.5 +scorer = 'internal' + +rounds = 8 + +competitors = ['p1', 'p2'] + +""" + +mcts_ctl = """ + +competition_type = 'mc_tuner' + +description = 'gomill_tests mc_tuner.' + +players = { + 'p1' : Player('test'), + } + +record_games = False +board_size = 9 +komi = 7.5 +candidate_colour = 'w' +opponent = 'p1' + +exploration_coefficient = 0.45 +initial_visits = 10 +initial_wins = 5 + +parameters = [ + Parameter('foo', + scale = LOG(0.01, 5.0), + split = 8, + format = 'I: %4.2f'), + ] + +def make_candidate(foo): + return Player('candidate') + +""" + +def test_get_job(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p2'] = Player('test sing song')", + ]) + job = fx.get_job() + tc.assertEqual(job.game_id, "0_000") + tc.assertEqual(job.game_data, ("0", 0)) + tc.assertEqual(job.board_size, 9) + tc.assertEqual(job.komi, 7.5) + tc.assertEqual(job.move_limit, 400) + tc.assertEqual(job.handicap, None) + tc.assertIs(job.handicap_is_free, False) + tc.assertIs(job.use_internal_scorer, True) + tc.assertEqual(job.sgf_game_name, 'test 0_000') + tc.assertEqual(job.sgf_event, 'test') + tc.assertIsNone(job.gtp_log_pathname) + tc.assertIsNone(job.sgf_filename) + tc.assertIsNone(job.sgf_dirname) + tc.assertIsNone(job.void_sgf_dirname) + tc.assertEqual(job.player_b.code, 'p1') + tc.assertEqual(job.player_w.code, 'p2') + tc.assertEqual(job.player_b.cmd_args, ['test']) + tc.assertEqual(job.player_w.cmd_args, ['test', 'sing', 'song']) + tc.assertDictEqual(job.player_b.gtp_aliases, {}) + tc.assertListEqual(job.player_b.startup_gtp_commands, []) + tc.assertEqual(job.stderr_pathname, "/nonexistent/ctl/test.log") + tc.assertIsNone(job.player_b.cwd) + tc.assertIsNone(job.player_b.environ) + tc.assertEqual(fx.ringmaster.games_in_progress, {'0_000': job}) + tc.assertEqual(fx.get_log(), + "starting game 0_000: p1 (b) vs p2 (w)\n") + tc.assertEqual(fx.get_history(), "") + + +def test_settings(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "handicap = 9", + "handicap_style = 'free'", + "record_games = True", + "scorer = 'players'" + ]) + fx.ringmaster.enable_gtp_logging() + job = fx.get_job() + tc.assertEqual(job.game_id, "0_000") + tc.assertEqual(job.handicap, 9) + tc.assertIs(job.handicap_is_free, True) + tc.assertIs(job.use_internal_scorer, False) + tc.assertEqual(job.stderr_pathname, "/nonexistent/ctl/test.log") + tc.assertEqual(job.gtp_log_pathname, + '/nonexistent/ctl/test.gtplogs/0_000.log') + tc.assertEqual(job.sgf_filename, '0_000.sgf') + tc.assertEqual(job.sgf_dirname, '/nonexistent/ctl/test.games') + tc.assertEqual(job.void_sgf_dirname, '/nonexistent/ctl/test.void') + tc.assertEqual(fx.ringmaster.get_sgf_filename("0_000"), "0_000.sgf") + tc.assertEqual(fx.ringmaster.get_sgf_pathname("0_000"), + "/nonexistent/ctl/test.games/0_000.sgf") + +def test_stderr_settings(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p2'] = Player('test', discard_stderr=True)", + ]) + job = fx.get_job() + tc.assertEqual(job.stderr_pathname, "/nonexistent/ctl/test.log") + tc.assertIs(job.player_b.discard_stderr, False) + tc.assertIs(job.player_w.discard_stderr, True) + +def test_stderr_settings_nolog(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p2'] = Player('test', discard_stderr=True)", + "stderr_to_log = False", + ]) + job = fx.get_job() + tc.assertIs(job.stderr_pathname, None) + tc.assertIs(job.player_b.discard_stderr, False) + tc.assertIs(job.player_w.discard_stderr, True) + + +def test_get_tournament_results(tc): + fx = Ringmaster_fixture(tc, playoff_ctl) + tc.assertRaisesRegexp(RingmasterError, "^status is not loaded$", + fx.ringmaster.get_tournament_results) + fx.initialise_clean() + tr = fx.ringmaster.get_tournament_results() + tc.assertEqual(tr.get_matchup_ids(), ['0']) + + fx2 = Ringmaster_fixture(tc, mcts_ctl) + fx2.initialise_clean() + tc.assertRaisesRegexp(RingmasterError, "^competition is not a tournament$", + fx2.ringmaster.get_tournament_results) + +def test_process_response(tc): + fx = Ringmaster_fixture(tc, playoff_ctl) + job = fx.get_job() + tc.assertEqual(fx.ringmaster.games_in_progress, {'0_000': job}) + tc.assertEqual( + fx.ringmaster.get_tournament_results().get_matchup_results('0'), []) + response = fake_response(job, 'w') + response.warnings = ['warningtest'] + response.log_entries = ['logtest'] + fx.ringmaster.process_response(response) + tc.assertEqual(fx.ringmaster.games_in_progress, {}) + tc.assertListEqual( + fx.messages('warnings'), + ["warningtest"]) + tc.assertListEqual( + fx.messages('results'), + ["game 0_000: p2 beat p1 W+1.5"]) + tc.assertEqual( + fx.ringmaster.get_tournament_results().get_matchup_results('0'), + [response.game_result]) + tc.assertEqual(fx.get_log(), + "starting game 0_000: p1 (b) vs p2 (w)\n" + "response from game 0_000\n" + "warningtest\n" + "logtest\n") + tc.assertEqual(fx.get_history(), "") + + +def test_check_players(tc): + fx = Ringmaster_fixture(tc, playoff_ctl) + tc.assertTrue(fx.ringmaster.check_players(discard_stderr=True)) + +def test_run(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + fx.initialise_clean() + fx.ringmaster.run(max_games=3) + tc.assertListEqual( + fx.messages('warnings'), + []) + tc.assertListEqual( + fx.messages('screen_report'), + ["p1 v p2 (3/400 games)\n" + "board size: 9 komi: 7.5\n" + " wins\n" + "p1 3 100.00% (black)\n" + "p2 0 0.00% (white)"]) + tc.assertMultiLineEqual( + fx.get_log(), + "run started at *** with max_games 3\n" + "starting game 0_000: p1 (b) vs p2 (w)\n" + "response from game 0_000\n" + "starting game 0_001: p1 (b) vs p2 (w)\n" + "response from game 0_001\n" + "starting game 0_002: p1 (b) vs p2 (w)\n" + "response from game 0_002\n" + "halting competition: max-games reached for this run\n" + "run finished at ***\n" + ) + tc.assertMultiLineEqual( + fx.get_history(), + " 0_000 p1 beat p2 B+10.5\n" + " 0_001 p1 beat p2 B+10.5\n" + " 0_002 p1 beat p2 B+10.5\n") + +def test_run_allplayall(tc): + fx = Ringmaster_fixture(tc, allplayall_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + fx.initialise_clean() + fx.ringmaster.run(max_games=3) + tc.assertListEqual( + fx.messages('warnings'), + []) + tc.assertListEqual( + fx.messages('screen_report'), + [dedent("""\ + 3/8 games played + + A B + A p1 2-1 + B p2 1-2""")]) + tc.assertMultiLineEqual( + fx.get_log(), + "run started at *** with max_games 3\n" + "starting game AvB_0: p1 (b) vs p2 (w)\n" + "response from game AvB_0\n" + "starting game AvB_1: p2 (b) vs p1 (w)\n" + "response from game AvB_1\n" + "starting game AvB_2: p1 (b) vs p2 (w)\n" + "response from game AvB_2\n" + "halting competition: max-games reached for this run\n" + "run finished at ***\n" + ) + tc.assertMultiLineEqual( + fx.get_history(), + " AvB_0 p1 beat p2 B+10.5\n" + " AvB_1 p2 beat p1 B+10.5\n" + " AvB_2 p1 beat p2 B+10.5\n") + +def test_check_players_fail(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p2'] = Player('test fail=startup')" + ]) + tc.assertFalse(fx.ringmaster.check_players(discard_stderr=True)) + +def test_run_fail(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test fail=startup', discard_stderr=True)", + ]) + fx.initialise_clean() + fx.ringmaster.run() + tc.assertListEqual( + fx.messages('warnings'), + ["game 0_000 -- aborting game due to error:\n" + "error starting subprocess for player p2:\n" + "exec forced to fail", + "halting run due to void games"]) + tc.assertListEqual( + fx.messages('screen_report'), + ["1 void games; see log file."]) + tc.assertMultiLineEqual( + fx.get_log(), + "run started at *** with max_games None\n" + "starting game 0_000: p1 (b) vs p2 (w)\n" + "game 0_000 -- aborting game due to error:\n" + "error starting subprocess for player p2:\n" + "exec forced to fail\n" + "halting competition: too many void games\n" + "run finished at ***\n") + tc.assertMultiLineEqual(fx.get_history(), "") + +def test_run_with_late_errors(tc): + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test init=fail_close', discard_stderr=True)", + ]) + def fail_close(channel): + channel.fail_close = True + fx.msf.register_init_callback('fail_close', fail_close) + fx.initialise_clean() + fx.ringmaster.run(max_games=2) + tc.assertListEqual(fx.messages('warnings'), []) + tc.assertMultiLineEqual( + fx.get_log(), + "run started at *** with max_games 2\n" + "starting game 0_000: p1 (b) vs p2 (w)\n" + "response from game 0_000\n" + "error closing player p2:\n" + "forced failure for close\n" + "starting game 0_001: p1 (b) vs p2 (w)\n" + "response from game 0_001\n" + "error closing player p2:\n" + "forced failure for close\n" + "halting competition: max-games reached for this run\n" + "run finished at ***\n") + tc.assertMultiLineEqual( + fx.get_history(), + " 0_000 p1 beat p2 B+10.5\n" + " 0_001 p1 beat p2 B+10.5\n") + +def test_status_roundtrip(tc): + fx1 = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + fx1.initialise_clean() + fx1.ringmaster.run(max_games=2) + tc.assertListEqual( + fx1.messages('warnings'), + []) + state = fx1.get_written_state() + + fx2 = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + fx2.initialise_with_state(state) + fx2.ringmaster.run(max_games=1) + tc.assertListEqual( + fx2.messages('warnings'), + []) + tc.assertListEqual( + fx2.messages('screen_report'), + ["p1 v p2 (3/400 games)\n" + "board size: 9 komi: 7.5\n" + " wins\n" + "p1 3 100.00% (black)\n" + "p2 0 0.00% (white)"]) + +def test_status(tc): + # Construct suitable competition status + fx1 = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + sfv = fx1.ringmaster.status_format_version + fx1.initialise_clean() + fx1.ringmaster.run(max_games=2) + competition_status = fx1.ringmaster.competition.get_status() + tc.assertListEqual( + fx1.messages('warnings'), + []) + status = { + 'void_game_count' : 0, + 'comp_vn' : fx1.ringmaster.competition.status_format_version, + 'comp' : competition_status, + } + + fx = Ringmaster_fixture(tc, playoff_ctl, [ + "players['p1'] = Player('test', discard_stderr=True)", + "players['p2'] = Player('test', discard_stderr=True)", + ]) + fx.initialise_with_state((sfv, status.copy())) + fx.ringmaster.run(max_games=1) + tc.assertListEqual( + fx.messages('warnings'), + []) + tc.assertListEqual( + fx.messages('screen_report'), + ["p1 v p2 (3/400 games)\n" + "board size: 9 komi: 7.5\n" + " wins\n" + "p1 3 100.00% (black)\n" + "p2 0 0.00% (white)"]) + + fx.ringmaster.set_test_status((-1, status.copy())) + tc.assertRaisesRegexp( + RingmasterError, + "incompatible status file", + fx.ringmaster.load_status) + + bad_status = status.copy() + del bad_status['void_game_count'] + fx.ringmaster.set_test_status((sfv, bad_status)) + tc.assertRaisesRegexp( + RingmasterError, + "incompatible status file: missing 'void_game_count'", + fx.ringmaster.load_status) + + bad_competition_status = competition_status.copy() + del bad_competition_status['results'] + bad_status_2 = status.copy() + bad_status_2['comp'] = bad_competition_status + fx.ringmaster.set_test_status((sfv, bad_status_2)) + tc.assertRaisesRegexp( + RingmasterError, + "error loading competition state: missing 'results'", + fx.ringmaster.load_status) + + bad_competition_status_2 = competition_status.copy() + bad_competition_status_2['scheduler'] = None + bad_status_3 = status.copy() + bad_status_3['comp'] = bad_competition_status_2 + fx.ringmaster.set_test_status((sfv, bad_status_3)) + tc.assertRaisesRegexp( + RingmasterError, + "error loading competition state:\n" + "AttributeError: 'NoneType' object has no attribute 'set_groups'", + fx.ringmaster.load_status) + + bad_status_4 = status.copy() + bad_status_4['comp_vn'] = -1 + fx.ringmaster.set_test_status((sfv, bad_status_4)) + tc.assertRaisesRegexp( + RingmasterError, + "incompatible status file", + fx.ringmaster.load_status) diff --git a/gomill/gomill_tests/run_gomill_testsuite.py b/gomill/gomill_tests/run_gomill_testsuite.py new file mode 100644 index 0000000..c85bd5c --- /dev/null +++ b/gomill/gomill_tests/run_gomill_testsuite.py @@ -0,0 +1,111 @@ +"""Construct and run the gomill testsuite.""" + +import sys +from optparse import OptionParser + +test_modules = [ + 'utils_tests', + 'common_tests', + 'board_tests', + 'sgf_grammar_tests', + 'sgf_properties_tests', + 'sgf_tests', + 'sgf_moves_tests', + 'gtp_engine_tests', + 'gtp_state_tests', + 'gtp_controller_tests', + 'gtp_proxy_tests', + 'gtp_game_tests', + 'game_job_tests', + 'setting_tests', + 'competition_scheduler_tests', + 'competition_tests', + 'playoff_tests', + 'allplayall_tests', + 'mcts_tuner_tests', + 'cem_tuner_tests', + 'ringmaster_tests', + ] + +def get_test_module(name): + """Import the specified gomill_tests module and return it.""" + dotted_name = "gomill_tests." + name + __import__(dotted_name) + return sys.modules[dotted_name] + +def get_test_modules(): + """Import all _tests modules in the specified order. + + Returns a list of module objects. + + """ + return [get_test_module(name) for name in test_modules] + +def run_testsuite(module_names, failfast, buffer): + """Run the gomill testsuite. + + module_names -- names of modules from gomill_tests, or None for all + failfast -- bool (stop at first failing test) + buffer -- bool (show stderr/stdout only for failing tests) + + Output is to stderr + + """ + try: + # This gives 'catchbreak' behaviour + unittest2.signals.installHandler() + except Exception: + pass + if module_names is None: + modules = get_test_modules() + else: + modules = [get_test_module(name) for name in module_names] + suite = unittest2.TestSuite() + for mdl in modules: + mdl.make_tests(suite) + runner = unittest2.TextTestRunner(failfast=failfast, buffer=buffer) + runner.run(suite) + +def run(argv): + parser = OptionParser(usage="%prog [options] [module] ...") + parser.add_option("-f", "--failfast", action="store_true", + help="stop after first test") + parser.add_option("-p", "--nobuffer", action="store_true", + help="show stderr/stdout for successful tests") + (options, args) = parser.parse_args(argv) + if args: + module_names = args + for module_name in module_names: + if module_name not in test_modules: + parser.error("unknown module: %s" % module_name) + else: + module_names = None + run_testsuite(module_names, options.failfast, not options.nobuffer) + +def import_unittest(): + """Import unittest2 into global scope. + + Raises NameError if it isn't available. + + Call this before using the functions in this module other than main(). + + """ + global unittest2 + try: + from gomill_tests.test_framework import unittest2 + except ImportError, e: + if hasattr(e, 'unittest2_missing'): + raise NameError("unittest2") + raise + +def main(): + try: + import_unittest() + except NameError: + sys.exit("gomill_tests: requires either Python 2.7 or " + "the 'unittest2' package") + run(sys.argv[1:]) + +if __name__ == "__main__": + main() + diff --git a/gomill/gomill_tests/setting_tests.py b/gomill/gomill_tests/setting_tests.py new file mode 100644 index 0000000..21c1629 --- /dev/null +++ b/gomill/gomill_tests/setting_tests.py @@ -0,0 +1,29 @@ +"""Tests for settings.py""" + +from gomill.settings import * + +from gomill_tests import gomill_test_support + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def test_interpret_shlex_sequence(tc): + iss = interpret_shlex_sequence + tc.assertEqual(iss("test"), ["test"]) + tc.assertEqual(iss("test "), ["test"]) + tc.assertEqual(iss("~test"), ["~test"]) + tc.assertEqual(iss("test foo bar"), ["test", "foo", "bar"]) + tc.assertEqual(iss("test 'foo bar'"), ["test", "foo bar"]) + tc.assertEqual(iss(u"test foo bar"), ["test", "foo", "bar"]) + tc.assertEqual(iss(["test"]), ["test"]) + tc.assertEqual(iss(["test", "foo", "bar"]), ["test", "foo", "bar"]) + tc.assertEqual(iss(["test", "foo bar"]), ["test", "foo bar"]) + tc.assertEqual(iss(("test", "foo", "bar")), ["test", "foo", "bar"]) + tc.assertRaisesRegexp(ValueError, "^empty$", iss, "") + tc.assertRaisesRegexp(ValueError, "^not a string or a sequence$", iss, None) + tc.assertRaisesRegexp(ValueError, "^element not a string$", + iss, ["test", None]) + tc.assertRaisesRegexp(ValueError, "^element contains NUL$", + iss, ["test", "fo\x00"]) + diff --git a/gomill/gomill_tests/sgf_grammar_tests.py b/gomill/gomill_tests/sgf_grammar_tests.py new file mode 100644 index 0000000..9e04624 --- /dev/null +++ b/gomill/gomill_tests/sgf_grammar_tests.py @@ -0,0 +1,362 @@ +"""Tests for sgf_grammar.py.""" + +from __future__ import with_statement + +from gomill_tests import gomill_test_support + +from gomill import sgf_grammar + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +def test_is_valid_property_identifier(tc): + ivpi = sgf_grammar.is_valid_property_identifier + tc.assertIs(ivpi("B"), True) + tc.assertIs(ivpi("PB"), True) + tc.assertIs(ivpi("ABCDEFGH"), True) + tc.assertIs(ivpi("ABCDEFGHI"), False) + tc.assertIs(ivpi(""), False) + tc.assertIs(ivpi("b"), False) + tc.assertIs(ivpi("Player"), False) + tc.assertIs(ivpi("P2"), False) + tc.assertIs(ivpi(" PB"), False) + tc.assertIs(ivpi("PB "), False) + tc.assertIs(ivpi("P B"), False) + tc.assertIs(ivpi("PB\x00"), False) + +def test_is_valid_property_value(tc): + ivpv = sgf_grammar.is_valid_property_value + tc.assertIs(ivpv(""), True) + tc.assertIs(ivpv("hello world"), True) + tc.assertIs(ivpv("hello\nworld"), True) + tc.assertIs(ivpv("hello \x00 world"), True) + tc.assertIs(ivpv("hello \xa3 world"), True) + tc.assertIs(ivpv("hello \xc2\xa3 world"), True) + tc.assertIs(ivpv("hello \\-) world"), True) + tc.assertIs(ivpv("hello (;[) world"), True) + tc.assertIs(ivpv("[hello world]"), False) + tc.assertIs(ivpv("hello ] world"), False) + tc.assertIs(ivpv("hello \\] world"), True) + tc.assertIs(ivpv("hello world \\"), False) + tc.assertIs(ivpv("hello world \\\\"), True) + tc.assertIs(ivpv("x" * 70000), True) + +def test_tokeniser(tc): + tokenise = sgf_grammar.tokenise + + tc.assertEqual(tokenise("(;B[ah][]C[a\xa3b])")[0], + [('D', '('), + ('D', ';'), + ('I', 'B'), + ('V', 'ah'), + ('V', ''), + ('I', 'C'), + ('V', 'a\xa3b'), + ('D', ')')]) + + def check_complete(s, *args): + tokens, tail_index = tokenise(s, *args) + tc.assertEqual(tail_index, len(s)) + return len(tokens) + + def check_incomplete(s, *args): + tokens, tail_index = tokenise(s, *args) + return len(tokens), tail_index + + # check surrounding junk + tc.assertEqual(check_complete(""), 0) + tc.assertEqual(check_complete("junk (;B[ah])"), 5) + tc.assertEqual(check_incomplete("junk"), (0, 0)) + tc.assertEqual(check_incomplete("junk (B[ah])"), (0, 0)) + tc.assertEqual(check_incomplete("(;B[ah]) junk"), (5, 8)) + + # check paren-balance count + tc.assertEqual(check_incomplete("(; ))(([ag]B C[ah])"), (3, 4)) + tc.assertEqual(check_incomplete("(;( )) (;)"), (5, 6)) + tc.assertEqual(check_incomplete("(;(()())) (;)"), (9, 9)) + + # check start_position + tc.assertEqual(check_complete("(; ))(;B[ah])", 4), 5) + tc.assertEqual(check_complete("(; ))junk (;B[ah])", 4), 5) + + tc.assertEqual(check_complete("(;XX[abc][def]KO[];B[bc])"), 11) + tc.assertEqual(check_complete("( ;XX[abc][def]KO[];B[bc])"), 11) + tc.assertEqual(check_complete("(; XX[abc][def]KO[];B[bc])"), 11) + tc.assertEqual(check_complete("(;XX [abc][def]KO[];B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc] [def]KO[];B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def] KO[];B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def]KO [];B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def]KO[] ;B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def]KO[]; B[bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def]KO[];B [bc])"), 11) + tc.assertEqual(check_complete("(;XX[abc][def]KO[];B[bc] )"), 11) + + tc.assertEqual(check_complete("( ;\nB\t[ah]\f[ef]\v)"), 6) + tc.assertEqual(check_complete("(;[Ran\xc2\xa3dom :\nstu@ff][ef]"), 4) + tc.assertEqual(check_complete("(;[ah)])"), 4) + + tc.assertEqual(check_incomplete("(;B[ag"), (3, 3)) + tc.assertEqual(check_incomplete("(;B[ag)"), (3, 3)) + tc.assertEqual(check_incomplete("(;AddBlack[ag])"), (3, 3)) + tc.assertEqual(check_incomplete("(;+B[ag])"), (2, 2)) + tc.assertEqual(check_incomplete("(;B+[ag])"), (3, 3)) + tc.assertEqual(check_incomplete("(;B[ag]+)"), (4, 7)) + + tc.assertEqual(check_complete(r"(;[ab \] cd][ef]"), 4) + tc.assertEqual(check_complete(r"(;[ab \] cd\\][ef]"), 4) + tc.assertEqual(check_complete(r"(;[ab \] cd\\\\][ef]"), 4) + tc.assertEqual(check_complete(r"(;[ab \] \\\] cd][ef]"), 4) + tc.assertEqual(check_incomplete(r"(;B[ag\])"), (3, 3)) + tc.assertEqual(check_incomplete(r"(;B[ag\\\])"), (3, 3)) + +def test_parser_structure(tc): + parse_sgf_game = sgf_grammar.parse_sgf_game + + def shape(s): + coarse_game = parse_sgf_game(s) + return len(coarse_game.sequence), len(coarse_game.children) + + tc.assertEqual(shape("(;C[abc]KO[];B[bc])"), (2, 0)) + tc.assertEqual(shape("initial junk (;C[abc]KO[];B[bc])"), (2, 0)) + tc.assertEqual(shape("(;C[abc]KO[];B[bc]) final junk"), (2, 0)) + tc.assertEqual(shape("(;C[abc]KO[];B[bc]) (;B[ag])"), (2, 0)) + + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"junk") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"()") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"(B[ag])") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"B[ag]") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_game, r"[ag]") + + tc.assertEqual(shape("(;C[abc]AB[ab][bc];B[bc])"), (2, 0)) + tc.assertEqual(shape("(;C[abc] AB[ab]\n[bc]\t;B[bc])"), (2, 0)) + tc.assertEqual(shape("(;C[abc]KO[];;B[bc])"), (3, 0)) + tc.assertEqual(shape("(;)"), (1, 0)) + + tc.assertRaisesRegexp(ValueError, "property with no values", + parse_sgf_game, r"(;B)") + tc.assertRaisesRegexp(ValueError, "unexpected value", + parse_sgf_game, r"(;[ag])") + tc.assertRaisesRegexp(ValueError, "unexpected value", + parse_sgf_game, r"(;[ag][ah])") + tc.assertRaisesRegexp(ValueError, "unexpected value", + parse_sgf_game, r"(;[B][ag])") + tc.assertRaisesRegexp(ValueError, "unexpected end of SGF data", + parse_sgf_game, r"(;B[ag]") + tc.assertRaisesRegexp(ValueError, "unexpected end of SGF data", + parse_sgf_game, r"(;B[ag][)]") + tc.assertRaisesRegexp(ValueError, "property with no values", + parse_sgf_game, r"(;B;W[ah])") + tc.assertRaisesRegexp(ValueError, "unexpected value", + parse_sgf_game, r"(;B[ag](;[ah]))") + tc.assertRaisesRegexp(ValueError, "property with no values", + parse_sgf_game, r"(;B W[ag])") + +def test_parser_tree_structure(tc): + parse_sgf_game = sgf_grammar.parse_sgf_game + + def shape(s): + coarse_game = parse_sgf_game(s) + return len(coarse_game.sequence), len(coarse_game.children) + + tc.assertEqual(shape("(;C[abc]AB[ab](;B[bc]))"), (1, 1)) + tc.assertEqual(shape("(;C[abc]AB[ab](;B[bc])))"), (1, 1)) + tc.assertEqual(shape("(;C[abc]AB[ab](;B[bc])(;B[bd]))"), (1, 2)) + + def shapetree(s): + def _shapetree(coarse_game): + return ( + len(coarse_game.sequence), + [_shapetree(pg) for pg in coarse_game.children]) + return _shapetree(parse_sgf_game(s)) + + tc.assertEqual(shapetree("(;C[abc]AB[ab](;B[bc])))"), + (1, [(1, [])]) + ) + tc.assertEqual(shapetree("(;C[abc]AB[ab](;B[bc]))))"), + (1, [(1, [])]) + ) + tc.assertEqual(shapetree("(;C[abc]AB[ab](;B[bc])(;B[bd])))"), + (1, [(1, []), (1, [])]) + ) + tc.assertEqual(shapetree(""" + (;C[abc]AB[ab];C[];C[] + (;B[bc]) + (;B[bd];W[ca] (;B[da])(;B[db];W[ea]) ) + )"""), + (3, [ + (1, []), + (2, [(1, []), (2, [])]) + ]) + ) + + tc.assertRaisesRegexp(ValueError, "unexpected end of SGF data", + parse_sgf_game, "(;B[ag];W[ah](;B[ai])") + tc.assertRaisesRegexp(ValueError, "empty sequence", + parse_sgf_game, "(;B[ag];())") + tc.assertRaisesRegexp(ValueError, "empty sequence", + parse_sgf_game, "(;B[ag]())") + tc.assertRaisesRegexp(ValueError, "empty sequence", + parse_sgf_game, "(;B[ag]((;W[ah])(;W[ai]))") + tc.assertRaisesRegexp(ValueError, "unexpected node", + parse_sgf_game, "(;B[ag];W[ah](;B[ai]);W[bd])") + tc.assertRaisesRegexp(ValueError, "property value outside a node", + parse_sgf_game, "(;B[ag];(W[ah];B[ai]))") + tc.assertRaisesRegexp(ValueError, "property value outside a node", + parse_sgf_game, "(;B[ag](;W[ah];)B[ai])") + tc.assertRaisesRegexp(ValueError, "property value outside a node", + parse_sgf_game, "(;B[ag](;W[ah])(B[ai]))") + +def test_parser_properties(tc): + parse_sgf_game = sgf_grammar.parse_sgf_game + + def props(s): + coarse_game = parse_sgf_game(s) + return coarse_game.sequence + + tc.assertEqual(props("(;C[abc]KO[]AB[ai][bh][ee];B[ bc])"), + [{'C': ['abc'], 'KO': [''], 'AB': ['ai', 'bh', 'ee']}, + {'B': [' bc']}]) + + tc.assertEqual(props(r"(;C[ab \] \) cd\\])"), + [{'C': [r"ab \] \) cd\\"]}]) + + tc.assertEqual(props("(;XX[1]YY[2]XX[3]YY[4])"), + [{'XX': ['1', '3'], 'YY' : ['2', '4']}]) + +def test_parse_sgf_collection(tc): + parse_sgf_collection = sgf_grammar.parse_sgf_collection + + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_collection, r"") + tc.assertRaisesRegexp(ValueError, "no SGF data found", + parse_sgf_collection, r"()") + + games = parse_sgf_collection("(;C[abc]AB[ab];X[];X[](;B[bc]))") + tc.assertEqual(len(games), 1) + tc.assertEqual(len(games[0].sequence), 3) + + games = parse_sgf_collection("(;X[1];X[2];X[3](;B[bc])) (;Y[1];Y[2])") + tc.assertEqual(len(games), 2) + tc.assertEqual(len(games[0].sequence), 3) + tc.assertEqual(len(games[1].sequence), 2) + + games = parse_sgf_collection( + "dummy (;X[1];X[2];X[3](;B[bc])) junk (;Y[1];Y[2]) Nonsense") + tc.assertEqual(len(games), 2) + tc.assertEqual(len(games[0].sequence), 3) + tc.assertEqual(len(games[1].sequence), 2) + + games = parse_sgf_collection( + "(( (;X[1];X[2];X[3](;B[bc])) ();) (;Y[1];Y[2]) )(Nonsense") + tc.assertEqual(len(games), 2) + tc.assertEqual(len(games[0].sequence), 3) + tc.assertEqual(len(games[1].sequence), 2) + + with tc.assertRaises(ValueError) as ar: + parse_sgf_collection( + "(( (;X[1];X[2];X[3](;B[bc])) ();) (;Y[1];Y[2]") + tc.assertEqual(str(ar.exception), + "error parsing game 1: unexpected end of SGF data") + + +def test_parse_compose(tc): + pc = sgf_grammar.parse_compose + tc.assertEqual(pc("word"), ("word", None)) + tc.assertEqual(pc("word:"), ("word", "")) + tc.assertEqual(pc("word:?"), ("word", "?")) + tc.assertEqual(pc("word:123"), ("word", "123")) + tc.assertEqual(pc("word:123:456"), ("word", "123:456")) + tc.assertEqual(pc(":123"), ("", "123")) + tc.assertEqual(pc(r"word\:more"), (r"word\:more", None)) + tc.assertEqual(pc(r"word\:more:?"), (r"word\:more", "?")) + tc.assertEqual(pc(r"word\\:more:?"), ("word\\\\", "more:?")) + tc.assertEqual(pc(r"word\\\:more:?"), (r"word\\\:more", "?")) + tc.assertEqual(pc("word\\\nmore:123"), ("word\\\nmore", "123")) + +def test_text_value(tc): + text_value = sgf_grammar.text_value + tc.assertEqual(text_value("abc "), "abc ") + tc.assertEqual(text_value("ab c"), "ab c") + tc.assertEqual(text_value("ab\tc"), "ab c") + tc.assertEqual(text_value("ab \tc"), "ab c") + tc.assertEqual(text_value("ab\nc"), "ab\nc") + tc.assertEqual(text_value("ab\\\nc"), "abc") + tc.assertEqual(text_value("ab\\\\\nc"), "ab\\\nc") + tc.assertEqual(text_value("ab\xa0c"), "ab\xa0c") + + tc.assertEqual(text_value("ab\rc"), "ab\nc") + tc.assertEqual(text_value("ab\r\nc"), "ab\nc") + tc.assertEqual(text_value("ab\n\rc"), "ab\nc") + tc.assertEqual(text_value("ab\r\n\r\nc"), "ab\n\nc") + tc.assertEqual(text_value("ab\r\n\r\n\rc"), "ab\n\n\nc") + tc.assertEqual(text_value("ab\\\r\nc"), "abc") + tc.assertEqual(text_value("ab\\\n\nc"), "ab\nc") + + tc.assertEqual(text_value("ab\\\tc"), "ab c") + + # These can't actually appear as SGF PropValues; anything sane will do + tc.assertEqual(text_value("abc\\"), "abc") + tc.assertEqual(text_value("abc]"), "abc]") + +def test_simpletext_value(tc): + simpletext_value = sgf_grammar.simpletext_value + tc.assertEqual(simpletext_value("abc "), "abc ") + tc.assertEqual(simpletext_value("ab c"), "ab c") + tc.assertEqual(simpletext_value("ab\tc"), "ab c") + tc.assertEqual(simpletext_value("ab \tc"), "ab c") + tc.assertEqual(simpletext_value("ab\nc"), "ab c") + tc.assertEqual(simpletext_value("ab\\\nc"), "abc") + tc.assertEqual(simpletext_value("ab\\\\\nc"), "ab\\ c") + tc.assertEqual(simpletext_value("ab\xa0c"), "ab\xa0c") + + tc.assertEqual(simpletext_value("ab\rc"), "ab c") + tc.assertEqual(simpletext_value("ab\r\nc"), "ab c") + tc.assertEqual(simpletext_value("ab\n\rc"), "ab c") + tc.assertEqual(simpletext_value("ab\r\n\r\nc"), "ab c") + tc.assertEqual(simpletext_value("ab\r\n\r\n\rc"), "ab c") + tc.assertEqual(simpletext_value("ab\\\r\nc"), "abc") + tc.assertEqual(simpletext_value("ab\\\n\nc"), "ab c") + + tc.assertEqual(simpletext_value("ab\\\tc"), "ab c") + + # These can't actually appear as SGF PropValues; anything sane will do + tc.assertEqual(simpletext_value("abc\\"), "abc") + tc.assertEqual(simpletext_value("abc]"), "abc]") + +def test_escape_text(tc): + tc.assertEqual(sgf_grammar.escape_text("abc"), "abc") + tc.assertEqual(sgf_grammar.escape_text(r"a\bc"), r"a\\bc") + tc.assertEqual(sgf_grammar.escape_text(r"ab[c]"), r"ab[c\]") + tc.assertEqual(sgf_grammar.escape_text(r"a\]bc"), r"a\\\]bc") + +def test_text_roundtrip(tc): + def roundtrip(s): + return sgf_grammar.text_value(sgf_grammar.escape_text(s)) + tc.assertEqual(roundtrip(r"abc"), r"abc") + tc.assertEqual(roundtrip(r"a\bc"), r"a\bc") + tc.assertEqual(roundtrip("abc\\"), "abc\\") + tc.assertEqual(roundtrip("ab]c"), "ab]c") + tc.assertEqual(roundtrip("abc]"), "abc]") + tc.assertEqual(roundtrip(r"abc\]"), r"abc\]") + tc.assertEqual(roundtrip("ab\nc"), "ab\nc") + tc.assertEqual(roundtrip("ab\n c"), "ab\n c") + + tc.assertEqual(roundtrip("ab\tc"), "ab c") + tc.assertEqual(roundtrip("ab\r\nc\n"), "ab\nc\n") + +def test_serialise_game_tree(tc): + serialised = ("(;AB[aa][ab][ac]C[comment \xa3];W[ab];C[];C[]" + "(;B[bc])(;B[bd];W[ca](;B[da])(;B[db];\n" + "W[ea])))\n") + coarse_game = sgf_grammar.parse_sgf_game(serialised) + tc.assertEqual(sgf_grammar.serialise_game_tree(coarse_game), serialised) + tc.assertEqual(sgf_grammar.serialise_game_tree(coarse_game, wrap=None), + serialised.replace("\n", "")+"\n") + diff --git a/gomill/gomill_tests/sgf_moves_tests.py b/gomill/gomill_tests/sgf_moves_tests.py new file mode 100644 index 0000000..b150ab7 --- /dev/null +++ b/gomill/gomill_tests/sgf_moves_tests.py @@ -0,0 +1,141 @@ +from gomill_tests import gomill_test_support + +from gomill import ascii_boards +from gomill import boards +from gomill import sgf +from gomill import sgf_moves + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +SAMPLE_SGF = """\ +(;AP[testsuite:0]CA[utf-8]DT[2009-06-06]FF[4]GM[1]KM[7.5]PB[Black engine] +PL[B]PW[White engine]RE[W+R]SZ[9]AB[ai][bh][ee]AW[fc][gc];B[dg];W[ef]C[comment +on two lines];B[];W[tt]C[Final comment]) +""" + +DIAGRAM1 = """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . o o . . +6 . . . . . . . . . +5 . . . . # . . . . +4 . . . . . . . . . +3 . . . . . . . . . +2 . # . . . . . . . +1 # . . . . . . . . + A B C D E F G H J\ +""" + +DIAGRAM2 = """\ +9 . . . . . . . . . +8 . . . . . . . . . +7 . . . . . . . . . +6 . . . . . . . . . +5 . . . . . . . . . +4 . . . . # . . . . +3 . . . . . . . . . +2 . . # . . . . . . +1 . . . . . . . . . + A B C D E F G H J\ +""" + + +def test_get_setup_and_moves(tc): + g1 = sgf.Sgf_game.from_string(SAMPLE_SGF) + board1, plays1 = sgf_moves.get_setup_and_moves(g1) + tc.assertDiagramEqual(ascii_boards.render_board(board1), DIAGRAM1) + tc.assertEqual(plays1, + [('b', (2, 3)), ('w', (3, 4)), ('b', None), ('w', None)]) + + g2 = sgf.Sgf_game(size=9) + root = g2.get_root() + root.set("AB", [(1, 2), (3, 4)]); + node = g2.extend_main_sequence() + node.set("B", (5, 6)) + node = g2.extend_main_sequence() + node.set("W", (5, 7)) + board2, plays2 = sgf_moves.get_setup_and_moves(g2) + tc.assertDiagramEqual(ascii_boards.render_board(board2), DIAGRAM2) + tc.assertEqual(plays2, + [('b', (5, 6)), ('w', (5, 7))]) + + g3 = sgf.Sgf_game.from_string("(;AB[ab][ba]AW[aa])") + tc.assertRaisesRegexp(ValueError, "setup position not legal", + sgf_moves.get_setup_and_moves, g3) + + g4 = sgf.Sgf_game.from_string("(;SZ[9];B[ab];AW[bc])") + tc.assertRaisesRegexp(ValueError, "setup properties after the root node", + sgf_moves.get_setup_and_moves, g4) + + g5 = sgf.Sgf_game.from_string("(;SZ[26];B[ab];W[bc])") + board5, plays5 = sgf_moves.get_setup_and_moves(g5) + tc.assertEqual(plays5, + [('b', (24, 0)), ('w', (23, 1))]) + + +def test_get_setup_and_moves_move_in_root(tc): + # A move in the root node is allowed (though deprecated) if there are no + # setup stones. + g1 = sgf.Sgf_game(size=9) + root = g1.get_root() + root.set("B", (1, 2)); + node = g1.extend_main_sequence() + node.set("W", (3, 4)) + board1, plays1 = sgf_moves.get_setup_and_moves(g1) + tc.assertTrue(board1.is_empty()) + tc.assertEqual(plays1, + [('b', (1, 2)), ('w', (3, 4))]) + + g2 = sgf.Sgf_game(size=9) + root = g2.get_root() + root.set("B", (1, 2)); + root.set("AW", [(3, 3)]); + node = g2.extend_main_sequence() + node.set("W", (3, 4)) + tc.assertRaisesRegexp(ValueError, "mixed setup and moves in root node", + sgf_moves.get_setup_and_moves, g2) + +def test_get_setup_and_moves_board_provided(tc): + b = boards.Board(9) + g1 = sgf.Sgf_game.from_string(SAMPLE_SGF) + board1, plays1 = sgf_moves.get_setup_and_moves(g1, b) + tc.assertIs(board1, b) + tc.assertDiagramEqual(ascii_boards.render_board(board1), DIAGRAM1) + tc.assertEqual(plays1, + [('b', (2, 3)), ('w', (3, 4)), ('b', None), ('w', None)]) + tc.assertRaisesRegexp(ValueError, "board not empty", + sgf_moves.get_setup_and_moves, g1, b) + b2 = boards.Board(19) + tc.assertRaisesRegexp(ValueError, "wrong board size, must be 9$", + sgf_moves.get_setup_and_moves, g1, b2) + + +def test_set_initial_position(tc): + board = ascii_boards.interpret_diagram(DIAGRAM1, 9) + sgf_game = sgf.Sgf_game(9) + sgf_moves.set_initial_position(sgf_game, board) + root = sgf_game.get_root() + tc.assertEqual(root.get("AB"), set([(0, 0), (1, 1), (4, 4)])) + tc.assertEqual(root.get("AW"), set([(6, 5), (6, 6)])) + tc.assertRaises(KeyError, root.get, 'AE') + +def test_indicate_first_player(tc): + g1 = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9];B[aa];W[ab])") + sgf_moves.indicate_first_player(g1) + tc.assertEqual(g1.serialise(), + "(;FF[4]GM[1]SZ[9];B[aa];W[ab])\n") + g2 = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9];W[aa];B[ab])") + sgf_moves.indicate_first_player(g2) + tc.assertEqual(g2.serialise(), + "(;FF[4]GM[1]PL[W]SZ[9];W[aa];B[ab])\n") + g3 = sgf.Sgf_game.from_string("(;AW[bc]FF[4]GM[1]SZ[9];B[aa];W[ab])") + sgf_moves.indicate_first_player(g3) + tc.assertEqual(g3.serialise(), + "(;FF[4]AW[bc]GM[1]PL[B]SZ[9];B[aa];W[ab])\n") + g4 = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9];C[no game])") + sgf_moves.indicate_first_player(g4) + tc.assertEqual(g4.serialise(), + "(;FF[4]GM[1]SZ[9];C[no game])\n") + diff --git a/gomill/gomill_tests/sgf_properties_tests.py b/gomill/gomill_tests/sgf_properties_tests.py new file mode 100644 index 0000000..e4f093a --- /dev/null +++ b/gomill/gomill_tests/sgf_properties_tests.py @@ -0,0 +1,386 @@ +"""Tests for sgf_properties.py.""" + +from textwrap import dedent + +from gomill_tests import gomill_test_support + +from gomill import sgf_properties + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + +def test_interpret_simpletext(tc): + def interpret(s, encoding): + context = sgf_properties._Context(19, encoding) + return sgf_properties.interpret_simpletext(s, context) + tc.assertEqual(interpret("a\nb\\\\c", "utf-8"), "a b\\c") + u = u"test \N{POUND SIGN}" + tc.assertEqual(interpret(u.encode("utf-8"), "UTF-8"), + u.encode("utf-8")) + tc.assertEqual(interpret(u.encode("iso-8859-1"), "ISO-8859-1"), + u.encode("utf-8")) + tc.assertRaises(UnicodeDecodeError, interpret, + u.encode("iso-8859-1"), "UTF-8") + tc.assertRaises(UnicodeDecodeError, interpret, u.encode("utf-8"), "ASCII") + +def test_serialise_simpletext(tc): + def serialise(s, encoding): + context = sgf_properties._Context(19, encoding) + return sgf_properties.serialise_simpletext(s, context) + tc.assertEqual(serialise("ab\\c", "utf-8"), "ab\\\\c") + u = u"test \N{POUND SIGN}" + tc.assertEqual(serialise(u.encode("utf-8"), "UTF-8"), + u.encode("utf-8")) + tc.assertEqual(serialise(u.encode("utf-8"), "ISO-8859-1"), + u.encode("iso-8859-1")) + tc.assertRaises(UnicodeEncodeError, serialise, + u"\N{EN DASH}".encode("utf-8"), "ISO-8859-1") + +def test_interpret_text(tc): + def interpret(s, encoding): + context = sgf_properties._Context(19, encoding) + return sgf_properties.interpret_text(s, context) + tc.assertEqual(interpret("a\nb\\\\c", "utf-8"), "a\nb\\c") + u = u"test \N{POUND SIGN}" + tc.assertEqual(interpret(u.encode("utf-8"), "UTF-8"), + u.encode("utf-8")) + tc.assertEqual(interpret(u.encode("iso-8859-1"), "ISO-8859-1"), + u.encode("utf-8")) + tc.assertRaises(UnicodeDecodeError, interpret, + u.encode("iso-8859-1"), "UTF-8") + tc.assertRaises(UnicodeDecodeError, interpret, u.encode("utf-8"), "ASCII") + +def test_serialise_text(tc): + def serialise(s, encoding): + context = sgf_properties._Context(19, encoding) + return sgf_properties.serialise_text(s, context) + tc.assertEqual(serialise("ab\\c", "utf-8"), "ab\\\\c") + u = u"test \N{POUND SIGN}" + tc.assertEqual(serialise(u.encode("utf-8"), "UTF-8"), + u.encode("utf-8")) + tc.assertEqual(serialise(u.encode("utf-8"), "ISO-8859-1"), + u.encode("iso-8859-1")) + tc.assertRaises(UnicodeEncodeError, serialise, + u"\N{EN DASH}".encode("utf-8"), "ISO-8859-1") + + +def test_interpret_number(tc): + interpret_number = sgf_properties.interpret_number + tc.assertEqual(interpret_number("1"), 1) + tc.assertIs(type(interpret_number("1")), int) + tc.assertEqual(interpret_number("0"), 0) + tc.assertEqual(interpret_number("-1"), -1) + tc.assertEqual(interpret_number("+1"), 1) + tc.assertRaises(ValueError, interpret_number, "1.5") + tc.assertRaises(ValueError, interpret_number, "0xaf") + tc.assertRaises(TypeError, interpret_number, 1) + + +def test_interpret_real(tc): + interpret_real = sgf_properties.interpret_real + tc.assertEqual(interpret_real("1"), 1.0) + tc.assertIs(type(interpret_real("1")), float) + tc.assertEqual(interpret_real("0"), 0.0) + tc.assertEqual(interpret_real("1.0"), 1.0) + tc.assertEqual(interpret_real("1.5"), 1.5) + tc.assertEqual(interpret_real("-1.5"), -1.5) + tc.assertEqual(interpret_real("+0.5"), 0.5) + tc.assertRaises(ValueError, interpret_real, "+") + tc.assertRaises(ValueError, interpret_real, "0xaf") + tc.assertRaises(ValueError, interpret_real, "inf") + tc.assertRaises(ValueError, interpret_real, "-inf") + tc.assertRaises(ValueError, interpret_real, "NaN") + tc.assertRaises(ValueError, interpret_real, "1e400") + #tc.assertRaises(TypeError, interpret_real, 1.0) + +def test_serialise_real(tc): + serialise_real = sgf_properties.serialise_real + tc.assertEqual(serialise_real(1), "1") + tc.assertEqual(serialise_real(-1), "-1") + tc.assertEqual(serialise_real(1.0), "1") + tc.assertEqual(serialise_real(-1.0), "-1") + tc.assertEqual(serialise_real(1.5), "1.5") + tc.assertEqual(serialise_real(-1.5), "-1.5") + tc.assertEqual(serialise_real(0.001), "0.001") + tc.assertEqual(serialise_real(0.0001), "0.0001") + tc.assertEqual(serialise_real(0.00001), "0") + tc.assertEqual(serialise_real(1e15), "1000000000000000") + tc.assertEqual(serialise_real(1e16), "10000000000000000") + tc.assertEqual(serialise_real(1e17), "100000000000000000") + tc.assertEqual(serialise_real(1e18), "1000000000000000000") + tc.assertEqual(serialise_real(-1e18), "-1000000000000000000") + # 1e400 is inf + tc.assertRaises(ValueError, serialise_real, 1e400) + # Python 2.5 returns 0 + #tc.assertRaises(ValueError, serialise_real, float("NaN")) + + +def test_interpret_move(tc): + def interpret_move(s, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_move(s, context) + tc.assertEqual(interpret_move("aa", 19), (18, 0)) + tc.assertEqual(interpret_move("ai", 19), (10, 0)) + tc.assertEqual(interpret_move("ba", 9), (8, 1)) + tc.assertEqual(interpret_move("tt", 21), (1, 19)) + tc.assertIs(interpret_move("tt", 19), None) + tc.assertIs(interpret_move("", 19), None) + tc.assertIs(interpret_move("", 21), None) + tc.assertRaises(ValueError, interpret_move, "Aa", 19) + tc.assertRaises(ValueError, interpret_move, "aA", 19) + tc.assertRaises(ValueError, interpret_move, "aaa", 19) + tc.assertRaises(ValueError, interpret_move, "a", 19) + tc.assertRaises(ValueError, interpret_move, "au", 19) + tc.assertRaises(ValueError, interpret_move, "ua", 19) + tc.assertRaises(ValueError, interpret_move, "a`", 19) + tc.assertRaises(ValueError, interpret_move, "`a", 19) + tc.assertRaises(ValueError, interpret_move, "11", 19) + tc.assertRaises(ValueError, interpret_move, " aa", 19) + tc.assertRaises(ValueError, interpret_move, "aa\x00", 19) + tc.assertRaises(TypeError, interpret_move, None, 19) + #tc.assertRaises(TypeError, interpret_move, ('a', 'a'), 19) + +def test_serialise_move(tc): + def serialise_move(s, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.serialise_move(s, context) + tc.assertEqual(serialise_move((18, 0), 19), "aa") + tc.assertEqual(serialise_move((10, 0), 19), "ai") + tc.assertEqual(serialise_move((8, 1), 19), "bk") + tc.assertEqual(serialise_move((8, 1), 9), "ba") + tc.assertEqual(serialise_move((1, 19), 21), "tt") + tc.assertEqual(serialise_move(None, 19), "tt") + tc.assertEqual(serialise_move(None, 20), "") + tc.assertRaises(ValueError, serialise_move, (3, 3), 0) + tc.assertRaises(ValueError, serialise_move, (3, 3), 27) + tc.assertRaises(ValueError, serialise_move, (9, 0), 9) + tc.assertRaises(ValueError, serialise_move, (-1, 0), 9) + tc.assertRaises(ValueError, serialise_move, (0, 9), 9) + tc.assertRaises(ValueError, serialise_move, (0, -1), 9) + tc.assertRaises(TypeError, serialise_move, (1, 1.5), 9) + +def test_interpret_point(tc): + def interpret_point(s, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_point(s, context) + tc.assertEqual(interpret_point("aa", 19), (18, 0)) + tc.assertEqual(interpret_point("ai", 19), (10, 0)) + tc.assertEqual(interpret_point("ba", 9), (8, 1)) + tc.assertEqual(interpret_point("tt", 21), (1, 19)) + tc.assertRaises(ValueError, interpret_point, "tt", 19) + tc.assertRaises(ValueError, interpret_point, "", 19) + tc.assertRaises(ValueError, interpret_point, "", 21) + tc.assertRaises(ValueError, interpret_point, "Aa", 19) + tc.assertRaises(ValueError, interpret_point, "aA", 19) + tc.assertRaises(ValueError, interpret_point, "aaa", 19) + tc.assertRaises(ValueError, interpret_point, "a", 19) + tc.assertRaises(ValueError, interpret_point, "au", 19) + tc.assertRaises(ValueError, interpret_point, "ua", 19) + tc.assertRaises(ValueError, interpret_point, "a`", 19) + tc.assertRaises(ValueError, interpret_point, "`a", 19) + tc.assertRaises(ValueError, interpret_point, "11", 19) + tc.assertRaises(ValueError, interpret_point, " aa", 19) + tc.assertRaises(ValueError, interpret_point, "aa\x00", 19) + tc.assertRaises(TypeError, interpret_point, None, 19) + #tc.assertRaises(TypeError, interpret_point, ('a', 'a'), 19) + +def test_serialise_point(tc): + def serialise_point(s, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.serialise_point(s, context) + tc.assertEqual(serialise_point((18, 0), 19), "aa") + tc.assertEqual(serialise_point((10, 0), 19), "ai") + tc.assertEqual(serialise_point((8, 1), 19), "bk") + tc.assertEqual(serialise_point((8, 1), 9), "ba") + tc.assertEqual(serialise_point((1, 19), 21), "tt") + tc.assertRaises(ValueError, serialise_point, None, 19) + tc.assertRaises(ValueError, serialise_point, None, 20) + tc.assertRaises(ValueError, serialise_point, (3, 3), 0) + tc.assertRaises(ValueError, serialise_point, (3, 3), 27) + tc.assertRaises(ValueError, serialise_point, (9, 0), 9) + tc.assertRaises(ValueError, serialise_point, (-1, 0), 9) + tc.assertRaises(ValueError, serialise_point, (0, 9), 9) + tc.assertRaises(ValueError, serialise_point, (0, -1), 9) + tc.assertRaises(TypeError, serialise_point, (1, 1.5), 9) + + +def test_interpret_point_list(tc): + def ipl(l, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_point_list(l, context) + tc.assertEqual(ipl([], 19), + set()) + tc.assertEqual(ipl(["aa"], 19), + set([(18, 0)])) + tc.assertEqual(ipl(["aa", "ai"], 19), + set([(18, 0), (10, 0)])) + tc.assertEqual(ipl(["ab:bc"], 19), + set([(16, 0), (16, 1), (17, 0), (17, 1)])) + tc.assertEqual(ipl(["ab:bc", "aa"], 19), + set([(18, 0), (16, 0), (16, 1), (17, 0), (17, 1)])) + # overlap is forbidden by the spec, but we accept it + tc.assertEqual(ipl(["aa", "aa"], 19), + set([(18, 0)])) + tc.assertEqual(ipl(["ab:bc", "bb:bc"], 19), + set([(16, 0), (16, 1), (17, 0), (17, 1)])) + # 1x1 rectangles are forbidden by the spec, but we accept them + tc.assertEqual(ipl(["aa", "bb:bb"], 19), + set([(18, 0), (17, 1)])) + # 'backwards' rectangles are forbidden by the spec, and we reject them + tc.assertRaises(ValueError, ipl, ["ab:aa"], 19) + tc.assertRaises(ValueError, ipl, ["ba:aa"], 19) + tc.assertRaises(ValueError, ipl, ["bb:aa"], 19) + + tc.assertRaises(ValueError, ipl, ["aa", "tt"], 19) + tc.assertRaises(ValueError, ipl, ["aa", ""], 19) + tc.assertRaises(ValueError, ipl, ["aa:", "aa"], 19) + tc.assertRaises(ValueError, ipl, ["aa:tt", "aa"], 19) + tc.assertRaises(ValueError, ipl, ["tt:aa", "aa"], 19) + +def test_compressed_point_list_spec_example(tc): + # Checks the examples at http://www.red-bean.com/sgf/DD_VW.html + def sgf_point(move, size): + row, col = move + row = size - row - 1 + col_s = "abcdefghijklmnopqrstuvwxy"[col] + row_s = "abcdefghijklmnopqrstuvwxy"[row] + return col_s + row_s + + def ipl(l, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_point_list(l, context) + tc.assertEqual( + set(sgf_point(move, 9) for move in ipl(["ac:ic"], 9)), + set(["ac", "bc", "cc", "dc", "ec", "fc", "gc", "hc", "ic"])) + tc.assertEqual( + set(sgf_point(move, 9) for move in ipl(["ae:ie"], 9)), + set(["ae", "be", "ce", "de", "ee", "fe", "ge", "he", "ie"])) + tc.assertEqual( + set(sgf_point(move, 9) for move in ipl(["aa:bi", "ca:ce"], 9)), + set(["aa", "ab", "ac", "ad", "ae", "af", "ag", "ah", "ai", + "bi", "bh", "bg", "bf", "be", "bd", "bc", "bb", "ba", + "ca", "cb", "cc", "cd", "ce"])) + +def test_serialise_point_list(tc): + def ipl(l, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_point_list(l, context) + def spl(l, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.serialise_point_list(l, context) + + tc.assertEqual(spl([(18, 0), (17, 1)], 19), ['aa', 'bb']) + tc.assertEqual(spl([(17, 1), (18, 0)], 19), ['aa', 'bb']) + tc.assertEqual(spl([], 9), []) + tc.assertEqual(ipl(spl([(1,2), (3,4), (4,5)], 19), 19), + set([(1,2), (3,4), (4,5)])) + tc.assertRaises(ValueError, spl, [(18, 0), None], 19) + + +def test_AP(tc): + def serialise(arg): + context = sgf_properties._Context(19, "UTF-8") + return sgf_properties.serialise_AP(arg, context) + def interpret(arg): + context = sgf_properties._Context(19, "UTF-8") + return sgf_properties.interpret_AP(arg, context) + + tc.assertEqual(serialise(("foo:bar", "2\n3")), "foo\\:bar:2\n3") + tc.assertEqual(interpret("foo\\:bar:2 3"), ("foo:bar", "2 3")) + tc.assertEqual(interpret("foo bar"), ("foo bar", "")) + +def test_ARLN(tc): + def serialise(arg, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.serialise_ARLN_list(arg, context) + def interpret(arg, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_ARLN_list(arg, context) + + tc.assertEqual(serialise([], 19), []) + tc.assertEqual(interpret([], 19), []) + tc.assertEqual(serialise([((7, 0), (5, 2)), ((4, 3), (2, 5))], 9), + ['ab:cd', 'de:fg']) + tc.assertEqual(interpret(['ab:cd', 'de:fg'], 9), + [((7, 0), (5, 2)), ((4, 3), (2, 5))]) + tc.assertRaises(ValueError, serialise, [((7, 0), None)], 9) + tc.assertRaises(ValueError, interpret, ['ab:tt', 'de:fg'], 9) + +def test_FG(tc): + def serialise(arg): + context = sgf_properties._Context(19, "UTF-8") + return sgf_properties.serialise_FG(arg, context) + def interpret(arg): + context = sgf_properties._Context(19, "UTF-8") + return sgf_properties.interpret_FG(arg, context) + tc.assertEqual(serialise(None), "") + tc.assertEqual(interpret(""), None) + tc.assertEqual(serialise((515, "th]is")), "515:th\\]is") + tc.assertEqual(interpret("515:th\\]is"), (515, "th]is")) + +def test_LB(tc): + def serialise(arg, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.serialise_LB_list(arg, context) + def interpret(arg, size): + context = sgf_properties._Context(size, "UTF-8") + return sgf_properties.interpret_LB_list(arg, context) + tc.assertEqual(serialise([], 19), []) + tc.assertEqual(interpret([], 19), []) + tc.assertEqual( + serialise([((6, 0), "lbl"), ((6, 1), "lb]l2")], 9), + ["ac:lbl", "bc:lb\\]l2"]) + tc.assertEqual( + interpret(["ac:lbl", "bc:lb\\]l2"], 9), + [((6, 0), "lbl"), ((6, 1), "lb]l2")]) + tc.assertRaises(ValueError, serialise, [(None, "lbl")], 9) + tc.assertRaises(ValueError, interpret, [':lbl', 'de:lbl2'], 9) + + +def test_presenter_interpret(tc): + p9 = sgf_properties.Presenter(9, "UTF-8") + p19 = sgf_properties.Presenter(19, "UTF-8") + tc.assertEqual(p9.interpret('KO', [""]), True) + tc.assertEqual(p9.interpret('SZ', ["9"]), 9) + tc.assertRaisesRegexp(ValueError, "multiple values", + p9.interpret, 'SZ', ["9", "blah"]) + tc.assertEqual(p9.interpret('CR', ["ab", "cd"]), set([(5, 2), (7, 0)])) + tc.assertRaises(ValueError, p9.interpret, 'SZ', []) + tc.assertRaises(ValueError, p9.interpret, 'CR', []) + tc.assertEqual(p9.interpret('DD', [""]), set()) + # all lists are treated like elists + tc.assertEqual(p9.interpret('CR', [""]), set()) + +def test_presenter_serialise(tc): + p9 = sgf_properties.Presenter(9, "UTF-8") + p19 = sgf_properties.Presenter(19, "UTF-8") + + tc.assertEqual(p9.serialise('KO', True), [""]) + tc.assertEqual(p9.serialise('SZ', 9), ["9"]) + tc.assertEqual(p9.serialise('KM', 3.5), ["3.5"]) + tc.assertEqual(p9.serialise('C', "foo\\:b]ar\n"), ["foo\\\\:b\\]ar\n"]) + tc.assertEqual(p19.serialise('B', (1, 2)), ["cr"]) + tc.assertEqual(p9.serialise('B', None), ["tt"]) + tc.assertEqual(p19.serialise('AW', set([(17, 1), (18, 0)])),["aa", "bb"]) + tc.assertEqual(p9.serialise('DD', [(1, 2), (3, 4)]), ["ch", "ef"]) + tc.assertEqual(p9.serialise('DD', []), [""]) + tc.assertRaisesRegexp(ValueError, "empty list", p9.serialise, 'CR', []) + tc.assertEqual(p9.serialise('AP', ("na:me", "2.3")), ["na\\:me:2.3"]) + tc.assertEqual(p9.serialise('FG', (515, "th]is")), ["515:th\\]is"]) + tc.assertEqual(p9.serialise('XX', "foo\\bar"), ["foo\\\\bar"]) + + tc.assertRaises(ValueError, p9.serialise, 'B', (1, 9)) + +def test_presenter_private_properties(tc): + p9 = sgf_properties.Presenter(9, "UTF-8") + tc.assertEqual(p9.serialise('XX', "9"), ["9"]) + tc.assertEqual(p9.interpret('XX', ["9"]), "9") + p9.set_private_property_type(p9.get_property_type("SZ")) + tc.assertEqual(p9.serialise('XX', 9), ["9"]) + tc.assertEqual(p9.interpret('XX', ["9"]), 9) + p9.set_private_property_type(None) + tc.assertRaisesRegexp(ValueError, "unknown property", + p9.serialise, 'XX', "foo\\bar") + tc.assertRaisesRegexp(ValueError, "unknown property", + p9.interpret, 'XX', ["asd"]) + diff --git a/gomill/gomill_tests/sgf_tests.py b/gomill/gomill_tests/sgf_tests.py new file mode 100644 index 0000000..df108c3 --- /dev/null +++ b/gomill/gomill_tests/sgf_tests.py @@ -0,0 +1,786 @@ +# -*- coding: utf-8 -*- +"""Tests for sgf.py.""" + +from __future__ import with_statement + +from textwrap import dedent + +from gomill_tests import gomill_test_support + +from gomill import sgf + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def test_new_sgf_game(tc): + g1 = sgf.Sgf_game(9) + tc.assertEqual(g1.get_size(), 9) + root = g1.get_root() + tc.assertEqual(root.get_raw('FF'), '4') + tc.assertEqual(root.get_raw('GM'), '1') + tc.assertEqual(root.get_raw('SZ'), '9') + tc.assertEqual(root.get_raw_property_map(), { + 'FF': ['4'], + 'GM': ['1'], + 'SZ': ['9'], + 'CA': ['UTF-8'], + }); + tc.assertEqual(list(root), []) + tc.assertEqual(root.parent, None) + tc.assertIs(root.owner, g1) + +def test_sgf_game_from_coarse_game_tree(tc): + class Namespace(object): + pass + coarse_game = Namespace() + coarse_game.sequence = [{'SZ' : ["9"]}, {'B' : ["aa"]}] + coarse_game.children = [] + g1 = sgf.Sgf_game.from_coarse_game_tree(coarse_game) + tc.assertEqual(g1.get_size(), 9) + root = g1.get_root() + tc.assertIs(root.get_raw_property_map(), coarse_game.sequence[0]) + tc.assertEqual(root.parent, None) + tc.assertIs(root.owner, g1) + tc.assertEqual(len(root), 1) + + coarse_game2 = Namespace() + coarse_game2.sequence = [{'SZ' : ["0"]}, {'B' : ["aa"]}] + coarse_game2.children = [] + tc.assertRaisesRegexp(ValueError, "size out of range: 0", + sgf.Sgf_game.from_coarse_game_tree, coarse_game2) + +def test_sgf_game_from_string(tc): + g1 = sgf.Sgf_game.from_string("(;)") + tc.assertEqual(g1.get_size(), 19) + tc.assertRaisesRegexp(ValueError, "unexpected end of SGF data", + sgf.Sgf_game.from_string, "(;SZ[9]") + g2 = sgf.Sgf_game.from_string("(;SZ[9])") + tc.assertEqual(g2.get_size(), 9) + tc.assertRaisesRegexp(ValueError, "bad SZ property: a", + sgf.Sgf_game.from_string, "(;SZ[a])") + tc.assertRaisesRegexp(ValueError, "size out of range: 27", + sgf.Sgf_game.from_string, "(;SZ[27])") + tc.assertRaisesRegexp(ValueError, "unknown encoding: $", + sgf.Sgf_game.from_string, "(;CA[])") + +def test_node(tc): + sgf_game = sgf.Sgf_game.from_string( + r"(;KM[6.5]C[sample\: comment]AB[ai][bh][ee]AE[];B[dg])") + node0 = sgf_game.get_root() + node1 = list(sgf_game.main_sequence_iter())[1] + tc.assertEqual(node0.get_size(), 19) + tc.assertEqual(node0.get_encoding(), "ISO-8859-1") + tc.assertIs(node0.has_property('KM'), True) + tc.assertIs(node0.has_property('XX'), False) + tc.assertIs(node1.has_property('KM'), False) + tc.assertEqual(set(node0.properties()), set(["KM", "C", "AB", "AE"])) + tc.assertEqual(set(node1.properties()), set(["B"])) + tc.assertEqual(node0.get_raw('C'), r"sample\: comment") + tc.assertEqual(node0.get_raw('AB'), "ai") + tc.assertEqual(node0.get_raw('AE'), "") + tc.assertRaises(KeyError, node0.get_raw, 'XX') + tc.assertEqual(node0.get_raw_list('KM'), ['6.5']) + tc.assertEqual(node0.get_raw_list('AB'), ['ai', 'bh', 'ee']) + tc.assertEqual(node0.get_raw_list('AE'), ['']) + tc.assertRaises(KeyError, node0.get_raw_list, 'XX') + tc.assertRaises(KeyError, node0.get_raw, 'XX') + +def test_property_combination(tc): + sgf_game = sgf.Sgf_game.from_string("(;XX[1]YY[2]XX[3]YY[4])") + node0 = sgf_game.get_root() + tc.assertEqual(node0.get_raw_list("XX"), ["1", "3"]) + tc.assertEqual(node0.get_raw_list("YY"), ["2", "4"]) + +def test_node_get(tc): + sgf_game = sgf.Sgf_game.from_string(dedent(r""" + (;AP[testsuite:0]CA[utf-8]DT[2009-06-06]FF[4]GM[1]KM[7.5]PB[Black engine] + PL[B]PW[White engine][xs]RE[W+R]SZ[9]AB[ai][bh][ee]AW[fd][gc]AE[]BM[2]VW[] + EV[Test + event] + C[123:\) + abc] + YY[none + sense] + ;B[dg]KO[]AR[ab:cd][de:fg]FG[515:first move] + LB[ac:lbl][bc:lbl2]) + """)) + root = sgf_game.get_root() + node1 = list(sgf_game.main_sequence_iter())[1] + tc.assertRaises(KeyError, root.get, 'XX') + tc.assertEqual(root.get('C'), "123:)\nabc") # Text + tc.assertEqual(root.get('EV'), "Test event") # Simpletext + tc.assertEqual(root.get('BM'), 2) # Double + tc.assertEqual(root.get('YY'), "none\nsense") # unknown (Text) + tc.assertIs(node1.get('KO'), True) # None + tc.assertEqual(root.get('KM'), 7.5) # Real + tc.assertEqual(root.get('GM'), 1) # Number + tc.assertEqual(root.get('PL'), 'b') # Color + tc.assertEqual(node1.get('B'), (2, 3)) # Point + tc.assertEqual(root.get('AB'), + set([(0, 0), (1, 1), (4, 4)])) # List of Point + tc.assertEqual(root.get('VW'), set()) # Empty elist + tc.assertEqual(root.get('AP'), ("testsuite", "0")) # Application + tc.assertEqual(node1.get('AR'), + [((7, 0), (5, 2)), ((4, 3), (2, 5))]) # Arrow + tc.assertEqual(node1.get('FG'), (515, "first move")) # Figure + tc.assertEqual(node1.get('LB'), + [((6, 0), "lbl"), ((6, 1), "lbl2")]) # Label + # Check we (leniently) treat lists like elists on read + tc.assertEqual(root.get('AE'), set()) + tc.assertRaisesRegexp(ValueError, "multiple values", root.get, 'PW') + +def test_text_values(tc): + def check(s): + sgf_game = sgf.Sgf_game.from_string(s) + return sgf_game.get_root().get("C") + # Round-trip check of Text values through tokeniser, parser, and + # text_value(). + tc.assertEqual(check(r"(;C[abc]KO[])"), r"abc") + tc.assertEqual(check(r"(;C[a\\bc]KO[])"), r"a\bc") + tc.assertEqual(check(r"(;C[a\\bc\]KO[])"), r"a\bc]KO[") + tc.assertEqual(check(r"(;C[abc\\]KO[])"), r"abc" + "\\") + tc.assertEqual(check(r"(;C[abc\\\]KO[])"), r"abc\]KO[") + tc.assertEqual(check(r"(;C[abc\\\\]KO[])"), r"abc" + "\\\\") + tc.assertEqual(check(r"(;C[abc\\\\\]KO[])"), r"abc\\]KO[") + tc.assertEqual(check(r"(;C[xxx :\) yyy]KO[])"), r"xxx :) yyy") + tc.assertEqual(check("(;C[ab\\\nc])"), "abc") + tc.assertEqual(check("(;C[ab\nc])"), "ab\nc") + + +SAMPLE_SGF = """\ +(;AP[testsuite:0]CA[utf-8]DT[2009-06-06]FF[4]GM[1]KM[7.5]PB[Black engine] +PL[B]PW[White engine]RE[W+R]SZ[9]AB[ai][bh][ee]AW[fc][gc];B[dg];W[ef]C[comment +on two lines];B[];W[tt]C[Final comment]) +""" + +SAMPLE_SGF_VAR = """\ +(;AP[testsuite:0]CA[utf-8]DT[2009-06-06]FF[4]GM[1]KM[7.5]PB[Black engine] +PL[B]RE[W+R]SZ[9]AB[ai][bh][ee]AW[fd][gc]VW[] +;B[dg] +;W[ef]C[comment +on two lines] +;B[] +;C[Nonfinal comment]VW[aa:bb] +(;B[ia];W[ib];B[ic]) +(;B[ib];W[ic] + (;B[id]) + (;B[ie]) +)) +""" + +def test_node_string(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF) + node = sgf_game.get_root() + tc.assertMultiLineEqual(str(node), dedent("""\ + AB[ai][bh][ee] + AP[testsuite:0] + AW[fc][gc] + CA[utf-8] + DT[2009-06-06] + FF[4] + GM[1] + KM[7.5] + PB[Black engine] + PL[B] + PW[White engine] + RE[W+R] + SZ[9] + """)) + +def test_node_get_move(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF) + nodes = list(sgf_game.main_sequence_iter()) + tc.assertEqual(nodes[0].get_move(), (None, None)) + tc.assertEqual(nodes[1].get_move(), ('b', (2, 3))) + tc.assertEqual(nodes[2].get_move(), ('w', (3, 4))) + tc.assertEqual(nodes[3].get_move(), ('b', None)) + tc.assertEqual(nodes[4].get_move(), ('w', None)) + +def test_node_get_setup_stones(tc): + sgf_game = sgf.Sgf_game.from_string( + r"(;KM[6.5]SZ[9]C[sample\: comment]AB[ai][bh][ee]AE[bb];B[dg])") + node0 = sgf_game.get_root() + node1 = list(sgf_game.main_sequence_iter())[1] + tc.assertIs(node0.has_setup_stones(), True) + tc.assertIs(node1.has_setup_stones(), False) + tc.assertEqual(node0.get_setup_stones(), + (set([(0, 0), (1, 1), (4, 4)]), set(), set([(7, 1)]))) + tc.assertEqual(node1.get_setup_stones(), + (set(), set(), set())) + +def test_sgf_game(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + nodes = list(sgf_game.main_sequence_iter()) + tc.assertEqual(sgf_game.get_size(), 9) + tc.assertEqual(sgf_game.get_komi(), 7.5) + tc.assertIs(sgf_game.get_handicap(), None) + tc.assertEqual(sgf_game.get_player_name('b'), "Black engine") + tc.assertIs(sgf_game.get_player_name('w'), None) + tc.assertEqual(sgf_game.get_winner(), 'w') + tc.assertEqual(nodes[2].get('C'), "comment\non two lines") + tc.assertEqual(nodes[4].get('C'), "Nonfinal comment") + + g2 = sgf.Sgf_game.from_string("(;)") + tc.assertEqual(g2.get_size(), 19) + tc.assertEqual(g2.get_komi(), 0.0) + tc.assertIs(g2.get_handicap(), None) + tc.assertIs(g2.get_player_name('b'), None) + tc.assertIs(g2.get_player_name('w'), None) + tc.assertEqual(g2.get_winner(), None) + +def test_tree_view(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + root = sgf_game.get_root() + tc.assertIsInstance(root, sgf.Tree_node) + tc.assertIs(root.parent, None) + tc.assertIs(root.owner, sgf_game) + tc.assertEqual(len(root), 1) + tc.assertEqual(root[0].get_raw('B'), "dg") + tc.assertTrue(root) + tc.assertEqual(root.index(root[0]), 0) + + branchnode = root[0][0][0][0] + tc.assertIsInstance(branchnode, sgf.Tree_node) + tc.assertIs(branchnode.parent, root[0][0][0]) + tc.assertIs(branchnode.owner, sgf_game) + tc.assertEqual(len(branchnode), 2) + tc.assertIs(branchnode[1], branchnode[-1]) + tc.assertEqual(branchnode[:1], [branchnode[0]]) + tc.assertEqual([node for node in branchnode], + [branchnode[0], branchnode[1]]) + with tc.assertRaises(IndexError): + branchnode[2] + tc.assertEqual(branchnode[0].get_raw('B'), "ia") + tc.assertEqual(branchnode[1].get_raw('B'), "ib") + tc.assertEqual(branchnode.index(branchnode[0]), 0) + tc.assertEqual(branchnode.index(branchnode[1]), 1) + + tc.assertEqual(len(branchnode[1][0]), 2) + + leaf = branchnode[1][0][1] + tc.assertIs(leaf.parent, branchnode[1][0]) + tc.assertEqual(len(leaf), 0) + tc.assertFalse(leaf) + + tc.assertIs(sgf_game.get_last_node(), root[0][0][0][0][0][0][0]) + + # check nothing breaks when first retrieval is by index + game2 = sgf.Sgf_game.from_string(SAMPLE_SGF) + root2 = game2.get_root() + tc.assertEqual(root2[0].get_raw('B'), "dg") + +def test_serialise(tc): + # Doesn't cover transcoding + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + serialised = sgf_game.serialise() + tc.assertEqual(serialised, dedent("""\ + (;FF[4]AB[ai][bh][ee]AP[testsuite:0]AW[fd][gc]CA[utf-8]DT[2009-06-06]GM[1] + KM[7.5]PB[Black engine]PL[B]RE[W+R]SZ[9]VW[];B[dg];C[comment + on two lines]W[ef] + ;B[];C[Nonfinal comment]VW[aa:bb](;B[ia];W[ib];B[ic])(;B[ib];W[ic](;B[id])(; + B[ie]))) + """)) + sgf_game2 = sgf.Sgf_game.from_string(serialised) + tc.assertEqual(map(str, sgf_game.get_main_sequence()), + map(str, sgf_game2.get_main_sequence())) + +def test_serialise_wrap(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + serialised = sgf_game.serialise(wrap=None) + tc.assertEqual(serialised, dedent("""\ + (;FF[4]AB[ai][bh][ee]AP[testsuite:0]AW[fd][gc]CA[utf-8]DT[2009-06-06]GM[1]KM[7.5]PB[Black engine]PL[B]RE[W+R]SZ[9]VW[];B[dg];C[comment + on two lines]W[ef];B[];C[Nonfinal comment]VW[aa:bb](;B[ia];W[ib];B[ic])(;B[ib];W[ic](;B[id])(;B[ie]))) + """)) + sgf_game2 = sgf.Sgf_game.from_string(serialised) + tc.assertEqual(map(str, sgf_game.get_main_sequence()), + map(str, sgf_game2.get_main_sequence())) + +def test_encoding(tc): + g1 = sgf.Sgf_game(19) + tc.assertEqual(g1.get_charset(), "UTF-8") + root = g1.get_root() + tc.assertEqual(root.get_encoding(), "UTF-8") + root.set("C", "£") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "£") + tc.assertEqual(g1.serialise(), dedent("""\ + (;FF[4]C[£]CA[UTF-8]GM[1]SZ[19]) + """)) + + g2 = sgf.Sgf_game(19, encoding="iso-8859-1") + tc.assertEqual(g2.get_charset(), "ISO-8859-1") + root = g2.get_root() + tc.assertEqual(root.get_encoding(), "ISO-8859-1") + root.set("C", "£") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "\xa3") + tc.assertEqual(g2.serialise(), dedent("""\ + (;FF[4]C[\xa3]CA[ISO-8859-1]GM[1]SZ[19]) + """)) + + tc.assertRaisesRegexp(ValueError, "unknown encoding: unknownencoding", + sgf.Sgf_game, 19, "unknownencoding") + + +def test_loaded_sgf_game_encoding(tc): + g1 = sgf.Sgf_game.from_string(""" + (;FF[4]C[£]CA[utf-8]GM[1]SZ[19]) + """) + tc.assertEqual(g1.get_charset(), "UTF-8") + root = g1.get_root() + tc.assertEqual(root.get_encoding(), "UTF-8") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "£") + tc.assertEqual(g1.serialise(), dedent("""\ + (;FF[4]C[£]CA[utf-8]GM[1]SZ[19]) + """)) + + g2 = sgf.Sgf_game.from_string(""" + (;FF[4]C[\xa3]CA[iso-8859-1]GM[1]SZ[19]) + """) + tc.assertEqual(g2.get_charset(), "ISO-8859-1") + root = g2.get_root() + tc.assertEqual(root.get_encoding(), "ISO-8859-1") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "\xa3") + tc.assertEqual(g2.serialise(), dedent("""\ + (;FF[4]C[\xa3]CA[iso-8859-1]GM[1]SZ[19]) + """)) + + g3 = sgf.Sgf_game.from_string(""" + (;FF[4]C[\xa3]GM[1]SZ[19]) + """) + tc.assertEqual(g3.get_charset(), "ISO-8859-1") + root = g3.get_root() + tc.assertEqual(root.get_encoding(), "ISO-8859-1") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "\xa3") + tc.assertEqual(g3.serialise(), dedent("""\ + (;FF[4]C[\xa3]GM[1]SZ[19]) + """)) + + # This is invalidly encoded. get() notices, but serialise() doesn't care. + g4 = sgf.Sgf_game.from_string(""" + (;FF[4]C[\xa3]CA[utf-8]GM[1]SZ[19]) + """) + tc.assertEqual(g4.get_charset(), "UTF-8") + root = g4.get_root() + tc.assertEqual(root.get_encoding(), "UTF-8") + tc.assertRaises(UnicodeDecodeError, root.get, "C") + tc.assertEqual(root.get_raw("C"), "\xa3") + tc.assertEqual(g4.serialise(), dedent("""\ + (;FF[4]C[\xa3]CA[utf-8]GM[1]SZ[19]) + """)) + + tc.assertRaisesRegexp( + ValueError, "unknown encoding: unknownencoding", + sgf.Sgf_game.from_string, """ + (;FF[4]CA[unknownencoding]GM[1]SZ[19]) + """) + +def test_override_encoding(tc): + g1 = sgf.Sgf_game.from_string(""" + (;FF[4]C[£]CA[iso-8859-1]GM[1]SZ[19]) + """, override_encoding="utf-8") + root = g1.get_root() + tc.assertEqual(root.get_encoding(), "UTF-8") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "£") + tc.assertEqual(g1.serialise(), dedent("""\ + (;FF[4]C[£]CA[UTF-8]GM[1]SZ[19]) + """)) + + g2 = sgf.Sgf_game.from_string(""" + (;FF[4]C[\xa3]CA[utf-8]GM[1]SZ[19]) + """, override_encoding="iso-8859-1") + root = g2.get_root() + tc.assertEqual(root.get_encoding(), "ISO-8859-1") + tc.assertEqual(root.get("C"), "£") + tc.assertEqual(root.get_raw("C"), "\xa3") + tc.assertEqual(g2.serialise(), dedent("""\ + (;FF[4]C[\xa3]CA[ISO-8859-1]GM[1]SZ[19]) + """)) + +def test_serialise_transcoding(tc): + g1 = sgf.Sgf_game.from_string(""" + (;FF[4]C[£]CA[utf-8]GM[1]SZ[19]) + """) + tc.assertEqual(g1.serialise(), dedent("""\ + (;FF[4]C[£]CA[utf-8]GM[1]SZ[19]) + """)) + g1.get_root().set("CA", "latin-1") + tc.assertEqual(g1.serialise(), dedent("""\ + (;FF[4]C[\xa3]CA[latin-1]GM[1]SZ[19]) + """)) + g1.get_root().set("CA", "unknown") + tc.assertRaisesRegexp(ValueError, "unsupported charset: \['unknown']", + g1.serialise) + + # improperly-encoded from the start + g2 = sgf.Sgf_game.from_string(""" + (;FF[4]C[£]CA[ascii]GM[1]SZ[19]) + """) + tc.assertEqual(g2.serialise(), dedent("""\ + (;FF[4]C[£]CA[ascii]GM[1]SZ[19]) + """)) + g2.get_root().set("CA", "utf-8") + tc.assertRaises(UnicodeDecodeError, g2.serialise) + + g3 = sgf.Sgf_game.from_string(""" + (;FF[4]C[Δ]CA[utf-8]GM[1]SZ[19]) + """) + g3.get_root().unset("CA") + tc.assertRaises(UnicodeEncodeError, g3.serialise) + +def test_tree_mutation(tc): + sgf_game = sgf.Sgf_game(9) + root = sgf_game.get_root() + n1 = root.new_child() + n1.set("N", "n1") + n2 = root.new_child() + n2.set("N", "n2") + n3 = n1.new_child() + n3.set("N", "n3") + n4 = root.new_child(1) + n4.set("N", "n4") + tc.assertEqual( + sgf_game.serialise(), + "(;FF[4]CA[UTF-8]GM[1]SZ[9](;N[n1];N[n3])(;N[n4])(;N[n2]))\n") + tc.assertEqual( + [node.get_raw_property_map() for node in sgf_game.main_sequence_iter()], + [node.get_raw_property_map() for node in root, root[0], n3]) + tc.assertIs(sgf_game.get_last_node(), n3) + + n1.delete() + tc.assertEqual( + sgf_game.serialise(), + "(;FF[4]CA[UTF-8]GM[1]SZ[9](;N[n4])(;N[n2]))\n") + tc.assertRaises(ValueError, root.delete) + +def test_tree_mutation_from_coarse_game(tc): + sgf_game = sgf.Sgf_game.from_string("(;SZ[9](;N[n1];N[n3])(;N[n2]))") + root = sgf_game.get_root() + n4 = root.new_child() + n4.set("N", "n4") + n3 = root[0][0] + tc.assertEqual(n3.get("N"), "n3") + n5 = n3.new_child() + n5.set("N", "n5") + tc.assertEqual(sgf_game.serialise(), + "(;SZ[9](;N[n1];N[n3];N[n5])(;N[n2])(;N[n4]))\n") + tc.assertEqual( + [node.get_raw_property_map() for node in sgf_game.main_sequence_iter()], + [node.get_raw_property_map() for node in root, root[0], n3, n5]) + tc.assertIs(sgf_game.get_last_node(), n5) + n3.delete() + tc.assertEqual(sgf_game.serialise(), + "(;SZ[9](;N[n1])(;N[n2])(;N[n4]))\n") + tc.assertRaises(ValueError, root.delete) + +def test_reparent(tc): + g1 = sgf.Sgf_game.from_string("(;SZ[9](;N[n1];N[n3])(;N[n2]))") + root = g1.get_root() + # Test with unexpanded root + tc.assertRaisesRegexp(ValueError, "would create a loop", + root.reparent, root) + n1 = root[0] + n2 = root[1] + n3 = root[0][0] + tc.assertEqual(n1.get("N"), "n1") + tc.assertEqual(n2.get("N"), "n2") + tc.assertEqual(n3.get("N"), "n3") + n3.reparent(n2) + tc.assertEqual(g1.serialise(), "(;SZ[9](;N[n1])(;N[n2];N[n3]))\n") + n3.reparent(n2) + tc.assertEqual(g1.serialise(), "(;SZ[9](;N[n1])(;N[n2];N[n3]))\n") + tc.assertRaisesRegexp(ValueError, "would create a loop", + root.reparent, n3) + tc.assertRaisesRegexp(ValueError, "would create a loop", + n3.reparent, n3) + g2 = sgf.Sgf_game(9) + tc.assertRaisesRegexp( + ValueError, "new parent doesn't belong to the same game", + n3.reparent, g2.get_root()) + +def test_reparent_index(tc): + g1 = sgf.Sgf_game.from_string("(;SZ[9](;N[n1];N[n3])(;N[n2]))") + root = g1.get_root() + n1 = root[0] + n2 = root[1] + n3 = root[0][0] + tc.assertEqual(n1.get("N"), "n1") + tc.assertEqual(n2.get("N"), "n2") + tc.assertEqual(n3.get("N"), "n3") + n3.reparent(root, index=1) + tc.assertEqual(g1.serialise(), "(;SZ[9](;N[n1])(;N[n3])(;N[n2]))\n") + n3.reparent(root, index=1) + tc.assertEqual(g1.serialise(), "(;SZ[9](;N[n1])(;N[n3])(;N[n2]))\n") + n3.reparent(root, index=2) + tc.assertEqual(g1.serialise(), "(;SZ[9](;N[n1])(;N[n2])(;N[n3]))\n") + +def test_extend_main_sequence(tc): + g1 = sgf.Sgf_game(9) + for i in xrange(6): + g1.extend_main_sequence().set("N", "e%d" % i) + tc.assertEqual( + g1.serialise(), + "(;FF[4]CA[UTF-8]GM[1]SZ[9];N[e0];N[e1];N[e2];N[e3];N[e4];N[e5])\n") + g2 = sgf.Sgf_game.from_string("(;SZ[9](;N[n1];N[n3])(;N[n2]))") + for i in xrange(6): + g2.extend_main_sequence().set("N", "e%d" % i) + tc.assertEqual( + g2.serialise(), + "(;SZ[9](;N[n1];N[n3];N[e0];N[e1];N[e2];N[e3];N[e4];N[e5])(;N[n2]))\n") + + +def test_get_sequence_above(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + root = sgf_game.get_root() + branchnode = root[0][0][0][0] + leaf = branchnode[1][0][1] + tc.assertEqual(sgf_game.get_sequence_above(root), []) + + tc.assertEqual(sgf_game.get_sequence_above(branchnode), + [root, root[0], root[0][0], root[0][0][0]]) + + tc.assertEqual(sgf_game.get_sequence_above(leaf), + [root, root[0], root[0][0], root[0][0][0], + branchnode, branchnode[1], branchnode[1][0]]) + + sgf_game2 = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + tc.assertRaisesRegexp(ValueError, "node doesn't belong to this game", + sgf_game2.get_sequence_above, leaf) + +def test_get_main_sequence_below(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + root = sgf_game.get_root() + branchnode = root[0][0][0][0] + leaf = branchnode[1][0][1] + tc.assertEqual(sgf_game.get_main_sequence_below(leaf), []) + + tc.assertEqual(sgf_game.get_main_sequence_below(branchnode), + [branchnode[0], branchnode[0][0], branchnode[0][0][0]]) + + tc.assertEqual(sgf_game.get_main_sequence_below(root), + [root[0], root[0][0], root[0][0][0], branchnode, + branchnode[0], branchnode[0][0], branchnode[0][0][0]]) + + sgf_game2 = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + tc.assertRaisesRegexp(ValueError, "node doesn't belong to this game", + sgf_game2.get_main_sequence_below, branchnode) + +def test_main_sequence(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + root = sgf_game.get_root() + + nodes = list(sgf_game.main_sequence_iter()) + tc.assertEqual(len(nodes), 8) + tc.assertIs(root.get_raw_property_map(), + nodes[0].get_raw_property_map()) + # Check that main_sequence_iter() optimisation has been used. + # (Have to call this before making the tree expand.) + with tc.assertRaises(AttributeError): + nodes[1].parent + + tree_nodes = sgf_game.get_main_sequence() + tc.assertEqual(len(tree_nodes), 8) + tc.assertIs(root.get_raw_property_map(), + tree_nodes[0].get_raw_property_map()) + tc.assertIs(tree_nodes[0], root) + tc.assertIs(tree_nodes[2].parent, tree_nodes[1]) + tc.assertIs(sgf_game.get_last_node(), tree_nodes[-1]) + + tree_node = root + for node in nodes: + tc.assertIs(tree_node.get_raw_property_map(), + node.get_raw_property_map()) + if tree_node: + tree_node = tree_node[0] + +def test_find(tc): + sgf_game = sgf.Sgf_game.from_string(SAMPLE_SGF_VAR) + root = sgf_game.get_root() + branchnode = root[0][0][0][0] + leaf = branchnode[1][0][1] + + tc.assertEqual(root.get("VW"), set()) + tc.assertIs(root.find("VW"), root) + tc.assertRaises(KeyError, root[0].get, "VW") + tc.assertEqual(root[0].find_property("VW"), set()) + tc.assertIs(root[0].find("VW"), root) + + tc.assertEqual(branchnode.get("VW"), + set([(7, 0), (7, 1), (8, 0), (8, 1)])) + tc.assertIs(branchnode.find("VW"), branchnode) + tc.assertEqual(branchnode.find_property("VW"), + set([(7, 0), (7, 1), (8, 0), (8, 1)])) + + tc.assertRaises(KeyError, leaf.get, "VW") + tc.assertIs(leaf.find("VW"), branchnode) + tc.assertEqual(leaf.find_property("VW"), + set([(7, 0), (7, 1), (8, 0), (8, 1)])) + + tc.assertIs(leaf.find("XX"), None) + tc.assertRaises(KeyError, leaf.find_property, "XX") + +def test_node_set_raw(tc): + sgf_game = sgf.Sgf_game.from_string(dedent(r""" + (;AP[testsuite:0]CA[utf-8]DT[2009-06-06]FF[4]GM[1]KM[7.5] + PB[Black engine]PW[White engine]RE[W+R]SZ[9] + AB[ai][bh][ee]AW[fd][gc]BM[2]VW[] + PL[B] + C[123abc] + ;B[dg]C[first move]) + """)) + root = sgf_game.get_root() + tc.assertEqual(root.get_raw('RE'), "W+R") + root.set_raw('RE', "W+2.5") + tc.assertEqual(root.get_raw('RE'), "W+2.5") + tc.assertRaises(KeyError, root.get_raw, 'XX') + root.set_raw('XX', "xyz") + tc.assertEqual(root.get_raw('XX'), "xyz") + + root.set_raw_list('XX', ("abc", "def")) + tc.assertEqual(root.get_raw('XX'), "abc") + tc.assertEqual(root.get_raw_list('XX'), ["abc", "def"]) + + tc.assertRaisesRegexp(ValueError, "empty property list", + root.set_raw_list, 'B', []) + + values = ["123", "456"] + root.set_raw_list('YY', values) + tc.assertEqual(root.get_raw_list('YY'), ["123", "456"]) + values.append("789") + tc.assertEqual(root.get_raw_list('YY'), ["123", "456"]) + + tc.assertRaisesRegexp(ValueError, "ill-formed property identifier", + root.set_raw, 'Black', "aa") + tc.assertRaisesRegexp(ValueError, "ill-formed property identifier", + root.set_raw_list, 'Black', ["aa"]) + + root.set_raw('C', "foo\\]bar") + tc.assertEqual(root.get_raw('C'), "foo\\]bar") + root.set_raw('C', "abc\\\\") + tc.assertEqual(root.get_raw('C'), "abc\\\\") + tc.assertRaisesRegexp(ValueError, "ill-formed raw property value", + root.set_raw, 'C', "foo]bar") + tc.assertRaisesRegexp(ValueError, "ill-formed raw property value", + root.set_raw, 'C', "abc\\") + tc.assertRaisesRegexp(ValueError, "ill-formed raw property value", + root.set_raw_list, 'C', ["abc", "de]f"]) + + root.set_raw('C', "foo\\]bar\\\nbaz") + tc.assertEqual(root.get('C'), "foo]barbaz") + + +def test_node_aliasing(tc): + # Check that node objects retrieved by different means use the same + # property map. + + sgf_game = sgf.Sgf_game.from_string(dedent(r""" + (;C[root];C[node 1]) + """)) + root = sgf_game.get_root() + plain_node = list(sgf_game.main_sequence_iter())[1] + tree_node = root[0] + # Check the main_sequence_iter() optimisation was used, otherwise this test + # isn't checking what it's supposed to. + tc.assertIsNot(tree_node, plain_node) + tc.assertIs(tree_node.__class__, sgf.Tree_node) + tc.assertIs(plain_node.__class__, sgf.Node) + + tc.assertEqual(tree_node.get_raw('C'), "node 1") + tree_node.set_raw('C', r"test\value") + tc.assertEqual(tree_node.get_raw('C'), r"test\value") + tc.assertEqual(plain_node.get_raw('C'), r"test\value") + + plain_node.set_raw_list('XX', ["1", "2", "3"]) + tc.assertEqual(tree_node.get_raw_list('XX'), ["1", "2", "3"]) + +def test_node_set(tc): + sgf_game = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9])") + root = sgf_game.get_root() + root.set("KO", True) + root.set("KM", 0.5) + root.set('DD', [(3, 4), (5, 6)]) + root.set('AB', set([(0, 0), (1, 1), (4, 4)])) + root.set('TW', set()) + root.set('XX', "nonsense [none]sense more n\\onsens\\e") + + tc.assertEqual(sgf_game.serialise(), dedent("""\ + (;FF[4]AB[ai][bh][ee]DD[ef][gd]GM[1]KM[0.5]KO[]SZ[9]TW[] + XX[nonsense [none\\]sense more n\\\\onsens\\\\e]) + """)) + +def test_node_unset(tc): + sgf_game = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9]HA[3])") + root = sgf_game.get_root() + tc.assertEqual(root.get('HA'), 3) + root.unset('HA') + tc.assertRaises(KeyError, root.unset, 'PL') + tc.assertEqual(sgf_game.serialise(), + "(;FF[4]GM[1]SZ[9])\n") + +def test_set_and_unset_size(tc): + g1 = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9]HA[3])") + root1 = g1.get_root() + tc.assertRaisesRegexp(ValueError, "changing size is not permitted", + root1.set, "SZ", 19) + root1.set("SZ", 9) + tc.assertRaisesRegexp(ValueError, "changing size is not permitted", + root1.unset, "SZ") + g2 = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[19]HA[3])") + root2 = g2.get_root() + root2.unset("SZ") + root2.set("SZ", 19) + +def test_set_and_unset_charset(tc): + g1 = sgf.Sgf_game.from_string("(;FF[4]CA[utf-8]GM[1]SZ[9]HA[3])") + tc.assertEqual(g1.get_charset(), "UTF-8") + root1 = g1.get_root() + root1.unset("CA") + tc.assertEqual(g1.get_charset(), "ISO-8859-1") + root1.set("CA", "iso-8859-1") + tc.assertEqual(g1.get_charset(), "ISO-8859-1") + root1.set("CA", "ascii") + tc.assertEqual(g1.get_charset(), "ASCII") + root1.set("CA", "unknownencoding") + tc.assertRaisesRegexp(ValueError, + "no codec available for CA unknownencoding", + g1.get_charset) + +def test_node_set_move(tc): + sgf_game = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9];B[aa];B[bb])") + root, n1, n2 = sgf_game.get_main_sequence() + tc.assertEqual(root.get_move(), (None, None)) + root.set_move('b', (1, 1)) + n1.set_move('w', (1, 2)) + n2.set_move('b', None) + tc.assertEqual(root.get('B'), (1, 1)) + tc.assertRaises(KeyError, root.get, 'W') + tc.assertEqual(n1.get('W'), (1, 2)) + tc.assertRaises(KeyError, n1.get, 'B') + tc.assertEqual(n2.get('B'), None) + tc.assertRaises(KeyError, n2.get, 'W') + +def test_node_setup_stones(tc): + sgf_game = sgf.Sgf_game.from_string("(;FF[4]GM[1]SZ[9]AW[aa:bb])") + root = sgf_game.get_root() + root.set_setup_stones( + [(1, 2), (3, 4)], + set(), + [(1, 3), (4, 5)], + ) + tc.assertEqual(root.get('AB'), set([(1, 2), (3, 4)])) + tc.assertRaises(KeyError, root.get, 'AW') + tc.assertEqual(root.get('AE'), set([(1, 3), (4, 5)])) + +def test_add_comment_text(tc): + sgf_game = sgf.Sgf_game(9) + root = sgf_game.get_root() + root.add_comment_text("hello\nworld") + tc.assertEqual(root.get('C'), "hello\nworld") + root.add_comment_text("hello\naga]in") + tc.assertEqual(root.get('C'), "hello\nworld\n\nhello\naga]in") + diff --git a/gomill/gomill_tests/subprocess_state_reporter.py b/gomill/gomill_tests/subprocess_state_reporter.py new file mode 100644 index 0000000..b4ef802 --- /dev/null +++ b/gomill/gomill_tests/subprocess_state_reporter.py @@ -0,0 +1,19 @@ +"""Fake GTP engine that reports info about cwd and environment. + +This is used by gtp_controller_tests.test_subprocess_channel + +This mustn't import any gomill or gomill_tests code. + +""" +import sys +import os + +def main(): + sys.stderr.write("subprocess_state_reporter: testing\n") + # Read the GTP command + sys.stdin.readline() + sys.stdout.write("= cwd: %s\nGOMILL_TEST:%s\n\n" % + (os.getcwd(), os.environ.get("GOMILL_TEST"))) + +if __name__ == "__main__": + main() diff --git a/gomill/gomill_tests/test_framework.py b/gomill/gomill_tests/test_framework.py new file mode 100644 index 0000000..8510c00 --- /dev/null +++ b/gomill/gomill_tests/test_framework.py @@ -0,0 +1,158 @@ +"""Generic (non-gomill-specific) test framework code.""" + +import sys + +if sys.version_info >= (2, 7): + import unittest as unittest2 +else: + try: + import unittest2 + except ImportError, e: + e.unittest2_missing = True + raise + +# This makes TestResult ignore lines from this module in tracebacks +__unittest = True + +class SupporterError(StandardError): + """Exception raised by support objects when something goes wrong. + + This is raised to indicate things like sequencing errors detected by mock + objects. + + """ + +class FrameworkTestCase(unittest2.TestCase): + """unittest2-style TestCase implementation with a few tweaks.""" + + # This is default in unittest2 but not python 2.7 unittest, so force it on. + longMessage = True + + def assertItemsEqual(self, expected_seq, actual_seq, msg=None): + """Variant implementation of standard assertItemsEqual. + + This uses the unorderable_list_difference check even if the lists are + sortable: I prefer its output. + + """ + expected = list(expected_seq) + actual = list(actual_seq) + missing, unexpected = unittest2.util.unorderable_list_difference( + expected, actual, ignore_duplicate=False + ) + errors = [] + if missing: + errors.append('Expected, but missing:\n %s' % + unittest2.util.safe_repr(missing)) + if unexpected: + errors.append('Unexpected, but present:\n %s' % + unittest2.util.safe_repr(unexpected)) + if errors: + standardMsg = '\n'.join(errors) + self.fail(self._formatMessage(msg, standardMsg)) + + +class SimpleTestCase(FrameworkTestCase): + """TestCase which runs a single function. + + Instantiate with the test function, which takes a TestCase parameter, eg: + def test_xxx(tc): + tc.assertEqual(2+2, 4) + + """ + + def __init__(self, fn): + FrameworkTestCase.__init__(self) + self.fn = fn + try: + self.name = fn.__module__.split(".", 1)[-1] + "." + fn.__name__ + except AttributeError: + self.name = str(fn) + + def runTest(self): + self.fn(self) + + def id(self): + return self.name + + def shortDescription(self): + return None + + def __str__(self): + return self.name + + def __repr__(self): + return "" % self.name + + +class ParameterisedTestCase(FrameworkTestCase): + """Parameterised testcase. + + Subclasses should define: + test_name -- short string + parameter_names -- list of identifiers + runTest + + """ + def __init__(self, code, *parameters): + FrameworkTestCase.__init__(self) + self.code = code + self.name = "%s.%s:%s" % (self.__class__.__module__.split(".", 1)[-1], + self.test_name, code) + for name, value in zip(self.parameter_names, parameters): + setattr(self, name, value) + + def runTest(self): + raise NotImplementedError + + def id(self): + return self.name + + def shortDescription(self): + return None + + def __str__(self): + return self.name + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.name) + + + +def _function_sort_key(fn): + try: + return fn.__code__.co_firstlineno + except AttributeError: + return str(fn) + +def make_simple_tests(source, prefix="test_", testcase_class=SimpleTestCase): + """Make test cases from a module's test_xxx functions. + + source -- dict (usually a module's globals()). + prefix -- string (default "test_") + testcase_class -- SimpleTestCase subclass to use + + Returns a list of TestCase objects. + + This makes a TestCase for each function in the values of 'source' whose + name begins with 'prefix'. + + The list is in the order of function definition (using the line number + attribute). + + """ + functions = [value for name, value in source.iteritems() + if name.startswith(prefix) and callable(value)] + functions.sort(key=_function_sort_key) + return [testcase_class(fn) for fn in functions] + + +class Fixture(object): + """A testing fixture. + + Instantiate fixture objects with a TestCase parameter. + + The fixture arranges for any necessary cleanup to be performed by calling + TestCase.addCleanUp. + + """ diff --git a/gomill/gomill_tests/test_support.py b/gomill/gomill_tests/test_support.py new file mode 100644 index 0000000..0dcf2ab --- /dev/null +++ b/gomill/gomill_tests/test_support.py @@ -0,0 +1,78 @@ +"""Generic (non-gomill-specific) test support code.""" + +import errno +from cStringIO import StringIO + +from gomill_tests.test_framework import SupporterError + +class Mock_writing_pipe(object): + """Mock writeable pipe object, with an interface like a cStringIO. + + If this is 'broken', it raises IOError(EPIPE) on any further writes. + + """ + def __init__(self): + self.sink = StringIO() + self.is_broken = False + + def write(self, s): + if self.is_broken: + raise IOError(errno.EPIPE, "Broken pipe") + try: + self.sink.write(s) + except ValueError, e: + raise IOError(errno.EIO, str(e)) + + def flush(self): + self.sink.flush() + + def close(self): + self.sink.close() + + def simulate_broken_pipe(self): + self.is_broken = True + + def getvalue(self): + return self.sink.getvalue() + + +class Mock_reading_pipe(object): + """Mock readable pipe object, with an interface like a cStringIO. + + Instantiate with the data to provide on the pipe. + + If this is 'broken', it always returns EOF from that point on. + + Set the attribute hangs_before_eof true to simulate a pipe that isn't closed + when it runs out of data. + + """ + def __init__(self, response): + self.source = StringIO(response) + self.is_broken = False + self.hangs_before_eof = False + + def read(self, n): + if self.is_broken: + return "" + result = self.source.read(n) + if self.hangs_before_eof and result == "": + raise SupporterError("read called with no data; this would hang") + return result + + def readline(self): + if self.is_broken: + return "" + result = self.source.readline() + if self.hangs_before_eof and not result.endswith("\n"): + raise SupporterError( + "readline called with no newline; this would hang") + return result + + def close(self): + self.source.close() + + def simulate_broken_pipe(self): + self.is_broken = True + + diff --git a/gomill/gomill_tests/utils_tests.py b/gomill/gomill_tests/utils_tests.py new file mode 100644 index 0000000..386e699 --- /dev/null +++ b/gomill/gomill_tests/utils_tests.py @@ -0,0 +1,63 @@ +"""Tests for utils.py.""" + +from gomill_tests import gomill_test_support + +from gomill import utils + +def make_tests(suite): + suite.addTests(gomill_test_support.make_simple_tests(globals())) + + +def test_format_float(tc): + ff = utils.format_float + tc.assertEqual(ff(1), "1") + tc.assertEqual(ff(1.0), "1") + tc.assertEqual(ff(1.5), "1.5") + +def test_format_percent(tc): + pct = utils.format_percent + tc.assertEqual(pct(1, 1), "100.00%") + tc.assertEqual(pct(1, 2), "50.00%") + tc.assertEqual(pct(1.0, 2.0), "50.00%") + tc.assertEqual(pct(1, 3), "33.33%") + tc.assertEqual(pct(0, 3), "0.00%") + tc.assertEqual(pct(2, 0), "??") + tc.assertEqual(pct(0, 0), "--") + +def test_sanitise_utf8(tc): + su = utils.sanitise_utf8 + tc.assertIsNone(su(None)) + tc.assertEqual(su(""), "") + tc.assertEqual(su("hello world"), "hello world") + s = u"test \N{POUND SIGN}".encode("utf-8") + tc.assertIs(su(s), s) + tc.assertEqual(su(u"test \N{POUND SIGN}".encode("latin1")), "test ?") + +def test_isinf(tc): + tc.assertIs(utils.isinf(0), False) + tc.assertIs(utils.isinf(0.0), False) + tc.assertIs(utils.isinf(3), False) + tc.assertIs(utils.isinf(3.0), False) + tc.assertIs(utils.isinf(1e300), False) + tc.assertIs(utils.isinf(1e400), True) + tc.assertIs(utils.isinf(-1e300), False) + tc.assertIs(utils.isinf(-1e400), True) + tc.assertIs(utils.isinf(1e-300), False) + tc.assertIs(utils.isinf(1e-400), False) + tc.assertIs(utils.isinf(float("inf")), True) + tc.assertIs(utils.isinf(float("-inf")), True) + tc.assertIs(utils.isinf(float("NaN")), False) + +def test_nan(tc): + tc.assertIs(utils.isnan(0), False) + tc.assertIs(utils.isnan(0.0), False) + tc.assertIs(utils.isnan(1e300), False) + tc.assertIs(utils.isnan(1e400), False) + tc.assertIs(utils.isnan(-1e300), False) + tc.assertIs(utils.isnan(-1e400), False) + tc.assertIs(utils.isnan(1e-300), False) + tc.assertIs(utils.isnan(1e-400), False) + tc.assertIs(utils.isnan(float("inf")), False) + tc.assertIs(utils.isnan(float("-inf")), False) + tc.assertIs(utils.isnan(float("NaN")), True) + diff --git a/gomill/ringmaster b/gomill/ringmaster new file mode 100755 index 0000000..d511839 --- /dev/null +++ b/gomill/ringmaster @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from gomill import ringmaster_command_line +ringmaster_command_line.main() diff --git a/gomill/setup.cfg b/gomill/setup.cfg new file mode 100644 index 0000000..11c21b8 --- /dev/null +++ b/gomill/setup.cfg @@ -0,0 +1,7 @@ +[debianize] +force-buildsystem: False +pycentral-backwards-compatibility: False + +[sdist_dsc] +force-buildsystem: False +pycentral-backwards-compatibility: False diff --git a/gomill/setup.py b/gomill/setup.py new file mode 100644 index 0000000..b3c9b94 --- /dev/null +++ b/gomill/setup.py @@ -0,0 +1,164 @@ +import glob +import imp +import os +import sys +from distutils import dir_util +from distutils.core import setup, Command + +VERSION = "0.7.2" + + +try: + from sphinx.setup_command import BuildDoc +except ImportError: + BuildDoc = None + +cmdclass = {} + +if BuildDoc: + cmdclass['build_sphinx'] = BuildDoc + + + +def find_script(name): + mode = os.F_OK | os.X_OK + for dirname in os.environ.get('PATH', os.defpath).split(os.pathsep): + if dirname == '': + continue + pathname = os.path.join(dirname, name) + if os.path.exists(pathname) and os.access(pathname, mode): + return pathname + return None + +def check_script(pathname): + s = open(pathname).read() + return 'from gomill import' in s + +def find_package(name): + try: + f, pathname, _ = imp.find_module(name) + except ImportError: + return None + if not isinstance(pathname, str): + return None + return os.path.realpath(pathname) + +def find_egg_info(package, pathname): + dirname = os.path.dirname(pathname) + return glob.glob(os.path.join(dirname, "%s-*.egg-info" % package)) + + +class uninstall(Command): + description = "uninstall the currently available version" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + files_to_remove = [] + dirs_to_remove = [] + + for script in self.distribution.scripts: + pathname = find_script(script) + if pathname is None: + self.warn("could not find script '%s'" % script) + continue + if check_script(pathname): + files_to_remove.append(pathname) + else: + self.warn("'%s' does not appear to be a gomill script; " + "not removing" % pathname) + + here = os.path.dirname(os.path.realpath(__file__)) + sys.path = [s for s in sys.path if os.path.realpath(s) != here] + + for package in self.distribution.packages: + pathname = find_package(package) + if pathname == here: + # belt and braces + pathname = None + if pathname is None: + self.warn("could not find package '%s'" % package) + continue + dirs_to_remove.append(pathname) + egg_infos = find_egg_info(package, pathname) + if len(egg_infos) > 1: + self.warn("multiple .egg-info files; not removing any:\n%s" + % "\n".join(egg_infos)) + egg_info_pathname = None + elif len(egg_infos) == 1: + pathname = egg_infos[0] + if os.path.isdir(pathname): + dirs_to_remove.append(pathname) + else: + files_to_remove.append(pathname) + + for pathname in files_to_remove: + self.execute(os.remove, (pathname,), "removing '%s'" % pathname) + for pathname in dirs_to_remove: + dir_util.remove_tree(pathname, dry_run=self.dry_run) + +cmdclass['uninstall'] = uninstall + + +GOMILL_URL = "http://mjw.woodcraft.me.uk/gomill/" + +LONG_DESCRIPTION = """\ +Gomill is a suite of tools, and a Python library, for use in developing and +testing Go-playing programs. It is based around the Go Text Protocol (GTP) and +the Smart Game Format (SGF). + +The principal tool is the ringmaster, which plays programs against each other +and keeps track of the results. + +There is also experimental support for automatically tuning program parameters. + +Download: http://mjw.woodcraft.me.uk/gomill/download/gomill-%(VERSION)s.tar.gz + +Documentation: http://mjw.woodcraft.me.uk/gomill/download/gomill-doc-%(VERSION)s.tar.gz + +Online Documentation: http://mjw.woodcraft.me.uk/gomill/doc/%(VERSION)s/ + +Changelog: http://mjw.woodcraft.me.uk/gomill/doc/%(VERSION)s/changes.html + +Git: http://mjw.woodcraft.me.uk/gomill/git/ + +Gitweb: http://mjw.woodcraft.me.uk/gitweb/gomill/ + +""" % vars() + +setup(name='gomill', + version=VERSION, + url=GOMILL_URL, + download_url="%sdownload/gomill-%s.tar.gz" % (GOMILL_URL, VERSION), + description="Tools for testing and tuning Go-playing programs", + long_description=LONG_DESCRIPTION, + author="Matthew Woodcraft", + author_email="matthew@woodcraft.me.uk", + packages=['gomill'], + scripts=['ringmaster'], + cmdclass=cmdclass, + classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python", + "Topic :: Games/Entertainment :: Board Games", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + keywords="go,baduk,weiqi,gtp,sgf", + license="MIT", + platforms="POSIX", + ) diff --git a/gomill/test_installed_gomill.py b/gomill/test_installed_gomill.py new file mode 100644 index 0000000..4ab2a05 --- /dev/null +++ b/gomill/test_installed_gomill.py @@ -0,0 +1,37 @@ +"""Run the gomill testsuite against an installed gomill package.""" + +import imp +import os +import sys + +# Remove the distribution directory from sys.path +if os.path.abspath(sys.path[0]) == os.path.abspath(os.path.dirname(__file__)): + del sys.path[0] + +try: + import gomill +except ImportError: + sys.exit("test_installed_gomill: can't find the gomill package") + +PACKAGE_NAME = "gomill_tests" + +# Make gomill_tests importable without the sibling gomill +def _make_newtests(): + dirpath = os.path.abspath( + os.path.join(os.path.dirname(__file__), PACKAGE_NAME)) + filepath = os.path.join(dirpath, "__init__.py") + # imp.load_source sets __name__ and __file__ + # __init__.py won't see its own __path__ set, but it doesn't contain any + # code, so it doesn't matter. + mdl = imp.load_source(PACKAGE_NAME, filepath) + assert mdl.__name__ == PACKAGE_NAME + mdl.__path__ = [dirpath] + mdl.__package__ = PACKAGE_NAME + sys.modules[PACKAGE_NAME] = mdl +_make_newtests() + +dirname = os.path.abspath(os.path.dirname(gomill.__file__)) +print >>sys.stderr, "testing gomill package in %s" % dirname +from gomill_tests import run_gomill_testsuite +run_gomill_testsuite.main() + diff --git a/lib/goban.py b/lib/goban.py index 5273cf3..c890d62 100644 --- a/lib/goban.py +++ b/lib/goban.py @@ -1,3 +1,6 @@ +import gomill.sgf + + class Goban: """Represents the go board. Handles stone placement, captures, etc""" @@ -11,12 +14,18 @@ class Goban: SCORING=6 - def __init__(self, board_size=19): + def __init__(self, board_size=19, file_name=None): # Build the board intersections self.board_size = board_size num_points = board_size * board_size self.board = [Goban.EMPTY] * num_points + self.file_name = file_name + self.sgf_game = None + + if self.file_name is not None: + self.load_sgf(file_name) + self.def_draw_codes = self._make_default_draw_codes() self.to_move = Goban.BLACK @@ -30,6 +39,23 @@ class Goban: self.winner = Goban.EMPTY + def load_sgf(self): + try: + with open(self.file_name, 'r') as fn: + self.sgf_game = sgf.Sgf_game.from_string(fn.read()) + except IOError: + # fixme - this should be convertable into a dialog box... perhaps it should throw an exception of its own + print 'There was a problem loading the SGF file.' + + for node in self.sgf_game.get_main_sequence(): + color, pos = node.get_move() + if color == 'b': + color = Goban.BLACK + elif color == 'w': + color = Goban.WHITE + self.play_move(color, pos) + + def set_hover(self, pos): rpos = self._real_pos(pos) if rpos == self.hover: diff --git a/pygo.py b/pygo.py index 75280b1..d61948e 100755 --- a/pygo.py +++ b/pygo.py @@ -6,6 +6,7 @@ import sys sys.path.append('lib/') sys.path.append('widgets/') +sys.path.append('gomill/') import goban import config @@ -50,6 +51,14 @@ class Pygo(): return True + def on_game_save(self, widget): + print 'stub: Pygo.on_game_save()' + + + def on_game_save_as(self, widget): + print 'stub: Pygo.on_game_save_as()' + + def on_game_close(self, widget): if self.games.get_current_page() == -1: return @@ -65,9 +74,12 @@ class Pygo(): game.winner_box.hide() - + def on_local_load_sgf(self, widget): + print 'stub: Pygo.on_local_load_sgf()' + + def on_net_direct(self, widget): - print 'stub: Pygo.on_menu_net_direct()' + print 'stub: Pygo.on_net_direct()' def on_quit(self, widget): diff --git a/ui/default.glade b/ui/default.glade index 95e58f5..41f599b 100644 --- a/ui/default.glade +++ b/ui/default.glade @@ -41,6 +41,24 @@ + + + True + gtk-save + True + + + + + + + True + gtk-save-as + True + + + + True @@ -81,6 +99,15 @@ + + + + True + _Load SGF + True + + + @@ -128,6 +155,7 @@ + 0