From 0e6804c9c7503f81e496ac878b6018734855fb88 Mon Sep 17 00:00:00 2001 From: AidenOchoa Date: Thu, 12 Nov 2026 15:11:33 -0700 Subject: [PATCH 1/6] Added skills from Lab5 and generated an intent.md file --- .claude/skills/check-vacuum/SKILL.md | 15 +++++++++ .../skills/compare-hyperparameters/SKILL.md | 18 +++++++++++ .claude/skills/verify-results/SKILL.md | 14 ++++++++ intent/intent.md | 32 +++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 .claude/skills/check-vacuum/SKILL.md create mode 100644 .claude/skills/compare-hyperparameters/SKILL.md create mode 100644 .claude/skills/verify-results/SKILL.md create mode 100644 intent/intent.md diff --git a/.claude/skills/check-vacuum/SKILL.md b/.claude/skills/check-vacuum/SKILL.md new file mode 100644 index 0000000..dcb5685 --- /dev/null +++ b/.claude/skills/check-vacuum/SKILL.md @@ -0,0 +1,15 @@ +--- +name: check-vacuum +description: Check the Lab 5 notebook and use focused specialists to review the environment, requirements, and code features. +disable-model-invocation: true +--- + +# Check Vacuum + +1. Work from the CS440-Lab5 repository root. Run `python scripts/check_project.py`. +2. Report errors and unfinished TODOs. A passing check does not prove that Q-learning works. +3. Invoke `environment-specialist` with `context/environment.md` and only the environment cell from `q_learning.ipynb`. Ask it to check movement, rewards, reset, and terminal handling. +4. Invoke `requirements-specialist` with `context/requirements.md` and the check summary. Ask it to identify missing Lab 5 deliverables. +5. Invoke `configuration-reviewer` with `.claude/settings.json`, this skill, and the three specialist definitions. Supply the checker output and only relevant learning-function excerpts if reviewing Q-learning. +6. Combine the specialists' short findings. Use actual subagent calls and report any invocation failure. +7. Return: checks run, hyperparameters, findings, and remaining work. Do not change files unless requested or invent training results. diff --git a/.claude/skills/compare-hyperparameters/SKILL.md b/.claude/skills/compare-hyperparameters/SKILL.md new file mode 100644 index 0000000..7a40fbe --- /dev/null +++ b/.claude/skills/compare-hyperparameters/SKILL.md @@ -0,0 +1,18 @@ +--- +name: compare-hyperparameters +description: Sweep alpha, epsilon, and gamma over several seeds and summarize the trade-offs, to support the hyperparameter explanation the lab asks for. +disable-model-invocation: true +argument-hint: "[--alpha ...] [--epsilon ...] [--gamma ...] [--seeds N]" +--- + +# Compare Hyperparameters + +1. Work from the CS440-Lab5 repository root. Run `python scripts/sweep_hyperparameters.py $ARGUMENTS`. With no arguments it sweeps alpha and epsilon around the notebook's current values over 5 seeds. Training seeds and evaluation seeds are disjoint from the notebook's own seeds. +2. Add `--out sweep_results.json` if the user wants the table saved. Otherwise do not write files. The script never changes the notebook or `results.json`. +3. Summarize the table: success rate (mean and range across seeds), average evaluation reward and steps, and late-training traps for each setting. +4. Explain the trade-offs only as far as the numbers show them: + - alpha: how fast values move versus how noisy they are. + - epsilon: exploration of rarely visited cells versus wasted or risky steps while training. + - gamma: how strongly the +10 goal outweighs the per-step cost. +5. Say when differences fall within the seed-to-seed range and cannot be told apart. Do not call any setting "best" or "optimal" on that basis alone. The sweep covers a small grid and a small number of seeds. +6. Return: the settings tried, the table, what the evidence supports, and a suggested wording for the README's hyperparameter explanation. Do not edit the README or the notebook parameters unless asked, and do not invent results. diff --git a/.claude/skills/verify-results/SKILL.md b/.claude/skills/verify-results/SKILL.md new file mode 100644 index 0000000..27e7b47 --- /dev/null +++ b/.claude/skills/verify-results/SKILL.md @@ -0,0 +1,14 @@ +--- +name: verify-results +description: Sanity-check results.json for consistency and compare the learned policy with a model-based optimum, then review the claims made about it. +disable-model-invocation: true +--- + +# Verify Results + +1. Work from the CS440-Lab5 repository root. If `results.json` is missing, say so and stop; do not create or invent results. Tell the user to run the notebook's training and recording cells. +2. Run `python scripts/verify_results.py`. It checks that outcome counts add up, that reward matches steps, and that evaluation is not better than the 6-step best case. It also compares each learned action with a value-iteration solution. That solution is used only for verification and is never given to the learner. +3. Report any `INCONSISTENT` lines first. Policy differences from optimal are findings, not failures. Note which cells differ and by how much, and that rarely visited cells are expected to be less accurate. +4. Read the Results section of `README.md` and check each number and claim against `results.json` and the script output. Flag any claim the output does not support, such as "optimal", "best", or an explanation of why a cell's action was chosen. +5. Note that one seed shows only that a configuration worked in that run, and that the policy is evaluated from the start cell only. +6. Return: the recorded timestamp, consistency result, cells that differ from optimal, README claims that need correcting, and what remains unverified. Do not edit files unless asked, and do not rerun training. diff --git a/intent/intent.md b/intent/intent.md new file mode 100644 index 0000000..553e007 --- /dev/null +++ b/intent/intent.md @@ -0,0 +1,32 @@ +# Vacuum Robot Q-Learning Project + +## Objective +Implement Q-learning for a vacuum robot in a 4x4 grid environment to navigate from the start position (0,0) to the reward location (3,3). + +## Environment Specifications +- **Grid Size**: 4x4 cells +- **Start Position**: (0,0) +- **Reward Location**: (3,3) - provides positive reward upon reaching +- **Obstacle**: Located at (1,1) - robot cannot move into this cell +- **Trap**: Located at (1,3) - incurs penalty if entered + +## Robot Actions +Available operations: +1. **Left** - Move one cell left +2. **Right** - Move one cell right +3. **Up** - Move one cell up +4. **Down** - Move one cell down +5. **Suck** - Clean the current cell (not used initially as environment is clean) +6. **NoOp** - Stay in place (incurs cost, not beneficial) + +## Constraints & Assumptions +- Environment starts clean, so Suck action is not needed initially +- NoOp action incurs a cost without progressing toward the goal, so it should be avoided +- Robot cannot move out of the environment or into obstacle cells +- Goal: Minimize cumulative cost to reach (3,3) using Q-learning + +## Implementation Requirements +- Implement Q-table initialization and updates +- Handle state transitions respecting obstacles and traps +- Apply appropriate action selection (e.g., epsilon-greedy) +- Track visited states for learning progress \ No newline at end of file From 7f2c31f10d25989f2f797d476b3cf652263a7349 Mon Sep 17 00:00:00 2001 From: Carson Loedel Date: Tue, 17 Nov 2026 12:00:57 -0700 Subject: [PATCH 2/6] added skill7 to the mini coding agenet --- mini_coding_agent.py | 227 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 222 insertions(+), 5 deletions(-) diff --git a/mini_coding_agent.py b/mini_coding_agent.py index 1bf2403..63e0ca8 100644 --- a/mini_coding_agent.py +++ b/mini_coding_agent.py @@ -12,7 +12,7 @@ DOC_NAMES = ("AGENTS.md", "README.md", "pyproject.toml", "package.json") -HELP_TEXT = "/help, /memory, /session, /reset, /exit" +HELP_TEXT = "/help, /memory, /session, /skills, /reset, /exit" WELCOME_ART = ( "/\\ /\\\\", "{ `---' }", @@ -27,6 +27,8 @@ "/help Show this help message.", "/memory Show the agent's distilled working memory.", "/session Show the path to the saved session file.", + "/skills List discovered skills and the skill log path.", + "/ Run a task with that skill explicitly.", "/reset Clear the current session history and memory.", "/exit Exit the agent.", ] @@ -34,6 +36,13 @@ MAX_TOOL_OUTPUT = 4000 MAX_HISTORY = 12000 IGNORED_PATH_NAMES = {".git", ".mini-coding-agent", "__pycache__", ".pytest_cache", ".ruff_cache", ".venv", "venv"} +SKILL_DIRS = ("skills", "skill", ".claude/skills") +SKILL_THRESHOLD = 0.25 +STOPWORDS = { + "a", "an", "and", "are", "as", "at", "be", "by", "can", "do", "for", "from", "how", "i", "in", + "is", "it", "me", "my", "of", "on", "or", "please", "so", "that", "the", "this", "to", "use", + "using", "we", "what", "with", "you", "your", +} ############################## #### Six Agent Components #### @@ -44,6 +53,7 @@ # 4) Context Reduction And Output Management -> clip, history_text # 5) Transcripts, Memory, And Resumption -> SessionStore, record, note_tool, ask, reset # 6) Delegation And Bounded Subagents -> tool_delegate +# 7) Agent Skills -> Skill, SkillRegistry, SkillLog, select_skill, activate_skill, tool_use_skill def now(): @@ -222,6 +232,129 @@ def complete(self, prompt, max_new_tokens): return data.get("response", "") +############################## +#### 7) Agent Skills ######### +############################## +class Skill: + def __init__(self, name, description, keywords, body, path): + self.name = name + self.description = description + self.keywords = keywords + self.body = body + self.path = path + + def match_text(self): + return " ".join([self.name.replace("-", " "), self.description, " ".join(self.keywords)]) + + +class SkillRegistry: + """Finds skill folders (skills//SKILL.md) and picks the one that fits a prompt.""" + + def __init__(self, skills, threshold=SKILL_THRESHOLD): + self.skills = {skill.name: skill for skill in skills} + self.threshold = threshold + + @classmethod + def discover(cls, root, threshold=SKILL_THRESHOLD): + root = Path(root) + skills = [] + for folder in SKILL_DIRS: + base = root / folder + if not base.is_dir(): + continue + for skill_dir in sorted(item for item in base.iterdir() if item.is_dir()): + for file_name in ("SKILL.md", "skill.md"): + path = skill_dir / file_name + if path.is_file(): + skills.append(cls.load(path)) + break + return cls(skills, threshold) + + @staticmethod + def load(path): + text = path.read_text(encoding="utf-8", errors="replace") + meta, body = {}, text + if text.startswith("---"): + parts = text.split("---", 2) + if len(parts) == 3: + for line in parts[1].splitlines(): + if ":" in line: + key, value = line.split(":", 1) + meta[key.strip().lower()] = value.strip().strip("\"'") + body = parts[2] + keywords = [word.strip() for word in meta.get("keywords", "").split(",") if word.strip()] + return Skill( + name=meta.get("name") or path.parent.name, + description=meta.get("description", ""), + keywords=keywords, + body=body.strip(), + path=path, + ) + + @staticmethod + def tokens(text): + words = re.findall(r"[a-z0-9]+", str(text).lower()) + result = [] + for word in words: + if word in STOPWORDS: + continue + for suffix in ("ing", "ed", "es", "s"): + if len(word) > len(suffix) + 2 and word.endswith(suffix): + word = word[: -len(suffix)] + break + result.append(word) + return result + + @classmethod + def similarity(cls, prompt, skill): + """Cosine similarity between word counts of the prompt and the skill's name/description/keywords.""" + left = cls.counts(cls.tokens(prompt)) + right = cls.counts(cls.tokens(skill.match_text())) + dot = sum(left[word] * right.get(word, 0) for word in left) + norm = (sum(v * v for v in left.values()) ** 0.5) * (sum(v * v for v in right.values()) ** 0.5) + return round(dot / norm, 3) if norm else 0.0 + + @staticmethod + def counts(words): + counts = {} + for word in words: + counts[word] = counts.get(word, 0) + 1 + return counts + + def match(self, prompt): + """Return (best_skill_or_None, scores, trigger). An explicit /skill-name always wins.""" + lowered = str(prompt).lower() + for name, skill in self.skills.items(): + if f"/{name.lower()}" in lowered: + return skill, {name: 1.0}, "explicit" + scores = {name: self.similarity(prompt, skill) for name, skill in self.skills.items()} + if not scores: + return None, scores, "none" + best = max(scores, key=scores.get) + if scores[best] >= self.threshold: + return self.skills[best], scores, "similarity" + return None, scores, "none" + + def text(self): + if not self.skills: + return "- none" + return "\n".join(f"- {skill.name}: {skill.description}" for skill in self.skills.values()) + + +class SkillLog: + """Appends one JSON line per event: prompt, similarity check, executed skill.""" + + def __init__(self, path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + + def write(self, event, **fields): + entry = {"time": now(), "event": event, **fields} + with self.path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + return entry + + class MiniAgent: def __init__( self, @@ -235,6 +368,7 @@ def __init__( depth=0, max_depth=1, read_only=False, + skills=None, ): self.model_client = model_client self.workspace = workspace @@ -253,6 +387,10 @@ def __init__( "history": [], "memory": {"task": "", "files": [], "notes": []}, } + # 7) Agent Skills: find skill folders and open the skill log. + self.skills = skills if skills is not None else SkillRegistry.discover(self.root) + self.skill_log = SkillLog(self.root / ".mini-coding-agent" / "skills.log") + self.active_skill = None self.tools = self.build_tools() self.prefix = self.build_prefix() self.session_path = self.session_store.save(self.session) @@ -325,6 +463,13 @@ def build_tools(self): "description": "Ask a bounded read-only child agent to investigate.", "run": self.tool_delegate, } + if self.skills.skills: + tools["use_skill"] = { + "schema": {"name": "str"}, + "risky": False, + "description": "Load a listed skill's instructions when the task needs that skill.", + "run": self.tool_use_skill, + } return tools ############################################ @@ -364,11 +509,14 @@ def build_prefix(self): "- New files should be complete and runnable, including obvious imports.", "- Do not repeat the same tool call with the same arguments if it did not help. Choose a different tool or return a final answer.", "- Required tool arguments must not be empty. Do not call read_file, write_file, patch_file, run_shell, or delegate with args={}.", + "- If an Active skill section is present, follow its instructions step by step.", + "- If the task needs one of the listed skills and none is active, call use_skill with its name.", ]) return "\n\n".join([ "You are Mini-Coding-Agent, a small local coding agent running through Ollama.", "Rules:\n" + rules, "Tools:\n" + tool_text, + "Skills:\n" + self.skills.text(), "Valid response examples:\n" + examples, self.workspace.text(), ]) @@ -420,13 +568,68 @@ def history_text(self): #### 2) Prompt Shape And Cache Reuse (Continued) ####### ######################################################## def prompt(self, user_message): - return "\n\n".join([ - self.prefix, - self.memory_text(), + sections = [self.prefix, self.memory_text()] + if self.active_skill: + sections.append(self.skill_text()) + sections += [ "Transcript:\n" + self.history_text(), "Current user request:\n" + user_message, + ] + return "\n\n".join(sections) + + ############################## + #### 7) Agent Skills ######### + ############################## + def skill_text(self): + skill = self.active_skill + return "\n".join([ + f"Active skill: {skill.name}", + f"- source: {skill.path.relative_to(self.root)}", + "- instructions:", + clip(skill.body, 3000), ]) + def select_skill(self, user_message): + """Log the prompt, run the similarity check, and activate the best skill if it passes.""" + self.skill_log.write("prompt", session=self.session["id"], prompt=user_message) + skill, scores, trigger = self.skills.match(user_message) + self.skill_log.write( + "similarity_check", + session=self.session["id"], + scores=scores, + threshold=self.skills.threshold, + best=max(scores, key=scores.get) if scores else None, + selected=skill.name if skill else None, + ) + score_text = ", ".join(f"{name}={score}" for name, score in sorted(scores.items(), key=lambda kv: -kv[1])) + print(f"[skills] similarity: {score_text or 'no skills found'} (threshold {self.skills.threshold})", file=sys.stderr) + if skill: + self.activate_skill(skill, trigger, scores.get(skill.name)) + else: + print("[skills] no skill selected", file=sys.stderr) + return skill + + def activate_skill(self, skill, trigger, score=None): + self.active_skill = skill + self.remember(self.session["memory"]["notes"], f"active skill: {skill.name}", 5) + self.skill_log.write( + "skill_executed", + session=self.session["id"], + skill=skill.name, + trigger=trigger, + score=score, + path=str(skill.path.relative_to(self.root)), + ) + print(f"[skills] executing skill '{skill.name}' (trigger: {trigger})", file=sys.stderr) + + def tool_use_skill(self, args): + name = str(args.get("name", "")).strip() + skill = self.skills.skills.get(name) + if skill is None: + raise ValueError(f"unknown skill '{name}'") + self.activate_skill(skill, "tool") + return f"skill '{skill.name}' loaded. Follow these instructions:\n{skill.body}" + ############################################### #### 5) Session Memory (Continued) ########### ############################################### @@ -447,6 +650,7 @@ def ask(self, user_message): if not memory["task"]: memory["task"] = clip(user_message.strip(), 300) self.record({"role": "user", "content": user_message, "created_at": now()}) + self.select_skill(user_message) tool_steps = 0 attempts = 0 @@ -530,6 +734,7 @@ def tool_example(self, name): "write_file": 'def binary_search(nums, target):\n return -1\n', "patch_file": 'return -1return mid', "delegate": '{"name":"delegate","args":{"task":"inspect README.md","max_steps":3}}', + "use_skill": '{"name":"use_skill","args":{"name":"pr-review"}}', } return examples.get(name, "") @@ -599,6 +804,12 @@ def validate_tool(self, name, args): raise ValueError("task must not be empty") return + if name == "use_skill": + skill_name = str(args.get("name", "")).strip() + if skill_name not in self.skills.skills: + raise ValueError(f"unknown skill '{skill_name}'; choose one of: {', '.join(self.skills.skills)}") + return + def approve(self, name, args): if self.read_only: return False @@ -717,6 +928,7 @@ def extract_raw(text, tag): def reset(self): self.session["history"] = [] self.session["memory"] = {"task": "", "files": [], "notes": []} + self.active_skill = None self.session_store.save(self.session) def path_is_within_root(self, resolved): @@ -860,6 +1072,7 @@ def tool_delegate(self, args): depth=self.depth + 1, max_depth=self.max_depth, read_only=True, + skills=self.skills, ) child.session["memory"]["task"] = task child.session["memory"]["notes"] = [clip(self.history_text(), 300)] @@ -1003,6 +1216,10 @@ def main(argv=None): if user_input == "/session": print(agent.session_path) continue + if user_input == "/skills": + print("Skills:\n" + agent.skills.text()) + print(f"Log: {agent.skill_log.path}") + continue if user_input == "/reset": agent.reset() print("session reset") @@ -1016,4 +1233,4 @@ def main(argv=None): if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file From 4458f52bc6771d9fc26a6b1f4e73e873ed681e3c Mon Sep 17 00:00:00 2001 From: AidenOchoa Date: Tue, 17 Nov 2026 15:23:44 -0700 Subject: [PATCH 3/6] Updated intent.md to match requirements of the assignment --- intent/intent.md | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/intent/intent.md b/intent/intent.md index 553e007..536877f 100644 --- a/intent/intent.md +++ b/intent/intent.md @@ -1,32 +1,9 @@ -# Vacuum Robot Q-Learning Project +# Coding Agent Skills -## Objective -Implement Q-learning for a vacuum robot in a 4x4 grid environment to navigate from the start position (0,0) to the reward location (3,3). - -## Environment Specifications -- **Grid Size**: 4x4 cells -- **Start Position**: (0,0) -- **Reward Location**: (3,3) - provides positive reward upon reaching -- **Obstacle**: Located at (1,1) - robot cannot move into this cell -- **Trap**: Located at (1,3) - incurs penalty if entered - -## Robot Actions -Available operations: -1. **Left** - Move one cell left -2. **Right** - Move one cell right -3. **Up** - Move one cell up -4. **Down** - Move one cell down -5. **Suck** - Clean the current cell (not used initially as environment is clean) -6. **NoOp** - Stay in place (incurs cost, not beneficial) +## Repository +This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. -## Constraints & Assumptions -- Environment starts clean, so Suck action is not needed initially -- NoOp action incurs a cost without progressing toward the goal, so it should be avoided -- Robot cannot move out of the environment or into obstacle cells -- Goal: Minimize cumulative cost to reach (3,3) using Q-learning - -## Implementation Requirements -- Implement Q-table initialization and updates -- Handle state transitions respecting obstacles and traps -- Apply appropriate action selection (e.g., epsilon-greedy) -- Track visited states for learning progress \ No newline at end of file +## Objective +The code agent must be able to identify a folder skill (such as .claude\skill\pr-review) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. +Each skill has a description which the agent will use to identify the right skill to use. +The new component must log the prompt, the similarity check, and the executed skill. From 6ec19397d5c8d5ae2f3c4265915c792dc9337b38 Mon Sep 17 00:00:00 2001 From: AidenOchoa Date: Tue, 17 Nov 2026 15:32:37 -0700 Subject: [PATCH 4/6] Added spec.md --- intent/spec.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 intent/spec.md diff --git a/intent/spec.md b/intent/spec.md new file mode 100644 index 0000000..f71ec1d --- /dev/null +++ b/intent/spec.md @@ -0,0 +1,22 @@ +# Coding Agent Skills Specification + +## Repository +This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. + +## Objective +The code agent must be able to identify a folder skill (such as `.claude\skill\pr-review`) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. +Each skill has a description which the agent will use to identify the right skill to use. +The new component must log the prompt, the similarity check, and the executed skill. + +## Requirements +1. **Skill Identification**: The agent should be able to detect when a user prompt or internal need requires a specific skill folder (e.g., `.claude\skill\pr-review`). +2. **Similarity Matching**: Implement a mechanism to compare the current context/prompt against skill descriptions to determine the best matching skill. +3. **Logging**: The system must log: + - The original prompt or trigger event + - The similarity score/result of the skill identification process + - The name and description of the executed skill + +## Implementation Notes +- Skills should be organized in folder structures within `.claude\skill\` +- Each skill folder contains a `skill.md` file with the skill's description and instructions +- The logging mechanism should be integrated into the agent's execution flow From c5a50b22aea730b8126bef1d27e406b76f15b8e8 Mon Sep 17 00:00:00 2001 From: AidenOchoa Date: Tue, 17 Nov 2026 15:54:50 -0700 Subject: [PATCH 5/6] Added plan.md and fixed typos --- intent/intent.md | 2 +- intent/plan.md | 197 +++++++++++++++++++++++++++++++++++++++++++++++ intent/spec.md | 6 +- 3 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 intent/plan.md diff --git a/intent/intent.md b/intent/intent.md index 536877f..9b37e3e 100644 --- a/intent/intent.md +++ b/intent/intent.md @@ -4,6 +4,6 @@ This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. ## Objective -The code agent must be able to identify a folder skill (such as .claude\skill\pr-review) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. +The code agent must be able to identify a folder skill (such as .claude\skills\pr-review) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. Each skill has a description which the agent will use to identify the right skill to use. The new component must log the prompt, the similarity check, and the executed skill. diff --git a/intent/plan.md b/intent/plan.md new file mode 100644 index 0000000..3619ba9 --- /dev/null +++ b/intent/plan.md @@ -0,0 +1,197 @@ +# Plan: Agent Skills for `mini_coding_agent.py` + +Derived from `spec.md` and a read of `mini_coding_agent.py` (1,019 lines, stdlib-only, Ollama backend, `MiniAgent` with six numbered components). + +## 1. Goal + +Let the agent discover skill folders (e.g. `.claude/skills/pr-review/skill.md`), pick the best one for a user prompt or a mid-run need by comparing against each skill's description, execute that skill's instructions, and log the prompt, the similarity result, and the executed skill (name + description). + +## 2. Design decisions (and why) + +| Decision | Choice | Reason grounded in the existing code | +|---|---|---| +| Dependencies | **None.** Pure-Python similarity, stdlib logging | The file only imports stdlib and talks to Ollama via `urllib`. Keep it single-file and dependency-free. | +| Similarity method | TF-IDF cosine over `name + description` tokens (with stopword removal and light suffix stripping). Matcher sits behind a small interface so an Ollama-embedding backend can be swapped in later. | Small corpus, small local model (default `qwen3.5:4b`), no extra service needed. Scores land in [0, 1], which makes thresholds and logs easy to read. | +| What "executing a skill" means | Make the skill's instructions the **active skill** for the current request, injected into the prompt as its own section. The model then follows them using the existing tools. | The agent has no separate execution engine; the model drives everything through `ask()`. | +| Where instructions are injected | A dedicated `Active skill:` section in `prompt()`, **not** the tool-result history | `history_text()` clips older tool output to 180 chars (900 for recent) and `run_tool()` clips to 4,000, so instructions delivered as a tool result would be truncated or dropped within a few steps. | +| Two trigger paths | (a) **Prompt trigger**: auto-match at the start of `ask()`. (b) **Internal-need trigger**: a new `use_skill` tool the model can call mid-run. | Spec: "when an appropriate prompt is triggered *or* when the code agent needs a specific skill during execution." | +| Log format | JSON Lines file at `/.mini-coding-agent/logs/skills.jsonl`, plus optional one-line stderr echo | Machine-parseable, sits next to the existing `sessions/` dir, and `.mini-coding-agent` is already in `IGNORED_PATH_NAMES`. | +| Folder name | Skills live **only** in `.claude/skills/`. Use `pathlib`, never hard-coded backslashes. | Confirmed convention. `Path` handles Windows and POSIX separators. | +| File name | Find `skill.md` case-insensitively (`SKILL.md`, `Skill.md`) | Spec says `skill.md`; Claude-style skills use `SKILL.md`. | + +## 3. Skill file format + +```markdown +--- +name: pr-review +description: Review a pull request or diff for bugs, style issues, and missing tests. +--- +# Instructions +1. Run `git diff ...HEAD`. +2. ... +``` + +- Parse a simple `key: value` frontmatter block with a small hand-written parser (no PyYAML). +- Fallbacks if frontmatter is missing: `name` = folder name, `description` = first non-heading paragraph. +- Skip (and log a warning for) skills with no usable description, since there would be nothing to match on. + +## 4. Implementation steps + +### Phase 0: Preparation + +- [ ] **0.1** Preserve the file's **CRLF line endings** when editing. `patch_file` / diffs that assume `\n` will fail to match. +- [ ] **0.2** Add a section header for the new component to match the existing convention, and extend the header comment block: `# 7) Agent Skills -> SkillRegistry, SkillMatcher, SkillLogger, use_skill`. +- [ ] **0.3** Add constants near `MAX_TOOL_OUTPUT`: + - `SKILLS_DIR = ".claude/skills"` + - `SKILL_FILE_NAME = "skill.md"` + - `MAX_SKILL_CHARS = 3000` (small local models have small context; clip instructions) + - `DEFAULT_SKILL_THRESHOLD = 0.20`, `DEFAULT_SKILL_MARGIN = 0.05` (starting values; tune in Phase 6) + - `MAX_SKILL_CATALOG = 20` + +### Phase 1: Skill model and discovery + +- [ ] **1.1** `Skill` dataclass (or plain class, matching the file's style): `name`, `description`, `instructions`, `path` (folder), `file` (the skill.md path). +- [ ] **1.2** `parse_skill_file(path)` implementing the format in section 3. Read with `encoding="utf-8", errors="replace"`, as the rest of the file does. +- [ ] **1.3** `SkillRegistry.discover(repo_root, cwd)`: + - Look under `SKILLS_DIR` in `repo_root` and `cwd` (mirroring how `WorkspaceContext.build` checks both for docs). + - One level of subfolders, each containing a `skill.md`. + - De-duplicate by skill name; the first one found wins, and log a warning on collisions. + - Return an empty registry (no error) if no skills dir exists, so existing behavior is unchanged. +- [ ] **1.4** Registry API: `all()`, `get(name)`, `__len__`, `catalog_text(limit=MAX_SKILL_CATALOG)` (name plus description clipped to ~160 chars). + +### Phase 2: Similarity matching + +- [ ] **2.1** `tokenize(text)`: lowercase, split on non-alphanumerics, drop a short stopword list and 1-char tokens, strip simple suffixes (`ing`, `ed`, `es`, `s`) so "reviewing" matches "review". +- [ ] **2.2** `SkillMatcher(skills)`: precompute TF-IDF vectors over `name + description`, with smoothed IDF so a 1-3 skill corpus doesn't produce all-zero weights. +- [ ] **2.3** `match(text) -> MatchResult` containing: + - `scores`: every skill with its score, sorted descending (logged in full) + - `best`: top skill or `None` + - `threshold`, `margin`, and `decision` (`"selected"` or `"no_match"`) + - `reason`: e.g. `below_threshold`, `ambiguous_margin`, `no_skills` +- [ ] **2.4** Selection rule: select the top skill only if `score >= threshold` **and** (`score - runner_up >= margin` or only one skill exists). Otherwise return no match. A weak guess is worse than no skill. +- [ ] **2.5** Define a `Matcher` interface (`match(text)`) so a future `OllamaEmbeddingMatcher` (`/api/embed`) can be selected by flag without touching the agent. + +### Phase 3: Logging + +- [ ] **3.1** `SkillLogger(path, session_id_getter, echo=False)` with `event(kind, **fields)` that appends one JSON line and (if `echo`) prints a short summary to **stderr** so stdout stays clean for answers. +- [ ] **3.2** Every record includes: `ts` (use the existing `now()`), `session_id`, `depth` (0 for the main agent, 1 for delegated child), `event`. +- [ ] **3.3** Three event types, one per spec requirement: + +| Event | Fields | Spec item | +|---|---|---| +| `skill_trigger` | `source` (`"prompt"` or `"tool"`), `text` (the original prompt or the model's stated need, clipped) | Original prompt or trigger event | +| `skill_match` | `candidates` (all name/score pairs), `best`, `best_score`, `threshold`, `margin`, `decision`, `reason`, `method` | Similarity score/result | +| `skill_executed` | `name`, `description`, `path`, `instructions_chars`, `truncated` (bool) | Name and description of executed skill | + +- [ ] **3.4** Also log `skill_skipped` when a match was rejected (below threshold or ambiguous) so no-match cases are visible, and `skill_warning` for discovery problems. +- [ ] **3.5** Logging must never crash the agent: wrap file writes in try/except and fall back to a stderr warning. +- [ ] **3.6** Log the prompt text clipped via `clip()` (e.g., 500 chars) since prompts may be long or contain secrets. Note this in the docs. + +### Phase 4: Agent integration + +All changes are in `MiniAgent` and `build_agent`. + +- [ ] **4.1 Constructor.** Add kwargs `skills=None`, `skill_matcher=None`, `skill_logger=None`. Assign them **before** `self.tools = self.build_tools()` and `self.prefix = self.build_prefix()`, because both will read the registry. Add `self.active_skill = None`. +- [ ] **4.2 Prefix catalog (`build_prefix`).** If the registry is non-empty, add a `Skills:` block (`registry.catalog_text()`) and a rule: *"If a listed skill fits a need, call `use_skill`."* This block is static for the session, so prompt-cache reuse (component 2) is preserved. +- [ ] **4.3 New tool `use_skill` (`build_tools`).** Register only when skills exist: + - schema `{"name": "str=''", "need": "str=''"}`, `risky: False`, description "Load a skill's instructions by name, or by describing the need." + - `tool_use_skill(args)`: resolve by exact `name`; else run the matcher on `need`. Log `skill_trigger(source="tool")` and `skill_match`, then activate. + - Returns a short confirmation (`"activated skill 'pr-review'; follow its instructions"`), **not** the full body (see section 2). +- [ ] **4.4 Validation and examples.** Add a `use_skill` branch to `validate_tool` (require non-empty `name` or `need`; unknown `name` is an error) and an entry in `tool_example`. Add an example line to `build_prefix`'s examples. Add `"name"` and `"need"` handling to `parse_xml_tool` only if XML form is wanted (JSON form works without changes). +- [ ] **4.5 Prompt trigger (`ask`).** At the top of `ask()`, after recording the user message and before the loop: + 1. Reset `self.active_skill = None`. + 2. Log `skill_trigger(source="prompt", text=user_message)`. + 3. Run `matcher.match(user_message)`; log `skill_match`. + 4. If `decision == "selected"`, call `self.activate_skill(skill)`; otherwise log `skill_skipped`. + - Run this **once per `ask()`**, not once per loop attempt, to avoid log spam and repeated work. +- [ ] **4.6 `activate_skill(skill, source)`.** Set `self.active_skill`, clip instructions to `MAX_SKILL_CHARS` (set `truncated`), log `skill_executed`, and `remember()` the skill name in session memory. +- [ ] **4.7 Prompt injection (`prompt`).** Insert an `Active skill:` section between `memory_text()` and `Transcript:`: + ``` + Active skill: pr-review + Description: ... + Follow these instructions for this request: + + ``` + Omit the section when no skill is active. +- [ ] **4.8 Mid-run switching.** If `use_skill` is called while a skill is already active, replace it and log both events. Cap `use_skill` calls per `ask()` (e.g., 2) so a confused model can't loop; the existing `repeated_tool_call` guard already blocks identical repeats. +- [ ] **4.9 Session state.** Add `"skill"` to `session["memory"]`, show it in `memory_text()`, clear it in `reset()`. Read with `.get("skill")` so sessions saved before this change still load via `from_session`. +- [ ] **4.10 Delegation (`tool_delegate`).** Pass `skills`, `skill_matcher`, and `skill_logger` to the child so it can use the same skills read-only. `depth` in the log distinguishes parent from child. The child's `ask()` will run its own prompt match on the delegated task, which is acceptable. +- [ ] **4.11 `build_agent`.** + 1. Build `SkillRegistry.discover(...)` from `workspace`. + 2. Build the matcher and `SkillLogger` at `/.mini-coding-agent/logs/skills.jsonl`. + 3. Pass all three into both `MiniAgent.from_session(...)` and `MiniAgent(...)`. + 4. Log any discovery warnings and a one-time `skills_loaded` summary (count and names). +- [ ] **4.12 Safety.** Skill instructions are untrusted repo content. They get **no** extra permissions: risky tools still go through `approve()`, and `read_only` children stay read-only. Skill-relative file reads still go through `path()`, so they are restricted to the repo root. + +### Phase 5: CLI and UX + +- [ ] **5.1** New arguments in `build_arg_parser`: `--skill-threshold` (float), `--no-skills` (disable the feature), `--skill-verbose` (echo log lines to stderr). +- [ ] **5.2** New `/skills` command in `main()` listing discovered skills and the currently active one. Update `HELP_DETAILS` (and `HELP_TEXT`). +- [ ] **5.3** Show skill count in `build_welcome` (e.g., a `SKILLS` cell) when any are loaded. +- [ ] **5.4** `/reset` also clears the active skill (covered by 4.9). + +### Phase 6: Tests + +Use the existing `FakeModelClient` for deterministic agent-loop tests, with a `tmp_path` workspace containing sample skills (`pr-review`, `write-tests`, `release-notes`). + +- [ ] **6.1 Parsing:** frontmatter, missing frontmatter fallback, empty description skipped, CRLF files, `SKILL.md` casing. +- [ ] **6.2 Discovery:** `.claude/skills/` under `cwd` vs `repo_root`, duplicate names, no skills dir (no-op). +- [ ] **6.3 Matching:** "review this PR" selects `pr-review`; unrelated prompt returns `no_match`; near-tie returns `ambiguous_margin`; scores are within [0, 1]; stemming works. +- [ ] **6.4 Logging:** a run produces `skill_trigger`, `skill_match`, `skill_executed` in order with the required fields; no-match produces `skill_skipped` and **no** `skill_executed`; a read-only log directory doesn't crash the agent. +- [ ] **6.5 Prompt trigger e2e:** Fake model returns ``; assert `agent.model_client.prompts[0]` contains the `Active skill:` section. +- [ ] **6.6 Tool trigger e2e:** Fake model emits `{"name":"use_skill",...}` then ``; assert the skill appears in the *next* prompt and the log has `source: "tool"`. +- [ ] **6.7 Regression:** with no skills dir (and with `--no-skills`), prompts are byte-identical to the pre-change output and `use_skill` is not in the tool list. +- [ ] **6.8 Session:** resume an old session JSON without `memory["skill"]`; `/reset` clears it. +- [ ] **6.9 Delegation:** child agent logs `depth: 1` and stays read-only. +- [ ] **6.10 Threshold tuning:** run the sample prompts against the sample skills, adjust `DEFAULT_SKILL_THRESHOLD` / `DEFAULT_SKILL_MARGIN` so the positive and negative cases above separate cleanly. + +### Phase 7: Documentation and samples + +- [ ] **7.1** Add a sample `.claude/skills/pr-review/skill.md` (also used as the test fixture). +- [ ] **7.2** README section: skill folder layout, file format, how matching works, the threshold flags, and the log schema with one example record per event type. +- [ ] **7.3** Document the caveat that prompts are logged (clipped) and the log file may contain sensitive text. + +## 5. Example log output + +```json +{"ts":"2026-11-17T18:02:11+00:00","session_id":"20261117-180211-a1b2c3","depth":0,"event":"skill_trigger","source":"prompt","text":"Please review my PR for bugs"} +{"ts":"...","event":"skill_match","method":"tfidf","candidates":[{"name":"pr-review","score":0.52},{"name":"write-tests","score":0.08}],"best":"pr-review","best_score":0.52,"threshold":0.2,"margin":0.05,"decision":"selected","reason":"ok"} +{"ts":"...","event":"skill_executed","name":"pr-review","description":"Review a pull request or diff for bugs, style issues, and missing tests.","path":".claude/skills/pr-review/skill.md","instructions_chars":1184,"truncated":false} +``` + +## 6. Files touched + +| File | Change | +|---|---| +| `mini_coding_agent.py` | New constants and classes (`Skill`, `SkillRegistry`, `SkillMatcher`, `SkillLogger`); edits to `MiniAgent.__init__`, `build_tools`, `build_prefix`, `prompt`, `ask`, `memory_text`, `reset`, `validate_tool`, `tool_example`, `tool_delegate`; edits to `build_agent`, `build_arg_parser`, `main`, `build_welcome` | +| `tests/test_skills.py` (new) | Phase 6 tests | +| `.claude/skills/pr-review/skill.md` (new) | Sample skill and fixture | +| `README.md` | Phase 7 docs | + +## 7. Requirement traceability + +| Spec requirement | Covered by | +|---|---| +| 1. Skill identification (prompt or internal need) | 1.3, 4.3, 4.5 | +| 2. Similarity matching against descriptions | 2.1-2.5 | +| 3a. Log original prompt / trigger event | 3.3 `skill_trigger`, 4.5, 4.3 | +| 3b. Log similarity score/result | 3.3 `skill_match`, 3.4 | +| 3c. Log name and description of executed skill | 3.3 `skill_executed`, 4.6 | +| Skills in `.claude/skills/`, each with `skill.md` | Section 3, 1.2, 1.3 | +| Logging integrated into execution flow | Phase 4 (logger passed through constructor, `ask`, `use_skill`, delegation) | + +## 8. Risks and open questions + +- **Small-model reliability.** A 4B model may rarely call `use_skill` on its own. The prompt-time auto-match (4.5) is the dependable path; the tool is a bonus. Consider lowering the threshold or adding the embedding matcher (2.5) if mid-run discovery matters. +- **Threshold sensitivity.** Lexical matching is weak on paraphrases ("look over my changes" vs. "review"). Mitigation: encourage keyword-rich descriptions in docs; revisit embeddings if misses are common. +- **Context budget.** Catalog plus active skill adds prompt length. Controlled by `MAX_SKILL_CATALOG`, description clipping, and `MAX_SKILL_CHARS`; may need lowering for the default model. +- **Scope.** Only project-level skills in `.claude/skills/` are supported. User-level skills (e.g. `~/.claude/skills`) are out of scope. +- **Skill-defined scripts.** Skills that tell the model to run commands still go through `run_shell` approval. Keep it that way. + +## 9. Suggested build order + +1. Phases 1-3 as pure, testable units (no agent changes), with their tests. +2. Phase 4.1-4.7 (prompt trigger path) and its e2e test. +3. Phase 4.3-4.4, 4.8 (`use_skill` tool path) and its e2e test. +4. Phase 4.9-4.11 (session, delegation, `build_agent`). +5. Phase 5 (CLI), then Phase 7 (docs), then Phase 6.10 threshold tuning against real prompts with Ollama. \ No newline at end of file diff --git a/intent/spec.md b/intent/spec.md index f71ec1d..b4ede10 100644 --- a/intent/spec.md +++ b/intent/spec.md @@ -4,12 +4,12 @@ This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. ## Objective -The code agent must be able to identify a folder skill (such as `.claude\skill\pr-review`) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. +The code agent must be able to identify a folder skill (such as `.claude\skills\pr-review`) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. Each skill has a description which the agent will use to identify the right skill to use. The new component must log the prompt, the similarity check, and the executed skill. ## Requirements -1. **Skill Identification**: The agent should be able to detect when a user prompt or internal need requires a specific skill folder (e.g., `.claude\skill\pr-review`). +1. **Skill Identification**: The agent should be able to detect when a user prompt or internal need requires a specific skill folder (e.g., `.claude\skills\pr-review`). 2. **Similarity Matching**: Implement a mechanism to compare the current context/prompt against skill descriptions to determine the best matching skill. 3. **Logging**: The system must log: - The original prompt or trigger event @@ -17,6 +17,6 @@ The new component must log the prompt, the similarity check, and the executed sk - The name and description of the executed skill ## Implementation Notes -- Skills should be organized in folder structures within `.claude\skill\` +- Skills should be organized in folder structures within `.claude\skills\` - Each skill folder contains a `skill.md` file with the skill's description and instructions - The logging mechanism should be integrated into the agent's execution flow From 8a379618ebba062116eb51440f86fc586c8ed371 Mon Sep 17 00:00:00 2001 From: GegeeTs Date: Thu, 19 Nov 2026 13:53:35 -0700 Subject: [PATCH 6/6] Refine agent skills and include the Q-learning demo --- README.md | 88 +- intent/intent.md | 5 +- intent/plan.md | 16 +- intent/spec.md | 11 +- mini_coding_agent.py | 56 +- .../.claude/agents/configuration-reviewer.md | 28 + .../.claude/agents/environment-specialist.md | 25 + .../.claude/agents/requirements-specialist.md | 25 + q-learning-9b/.claude/settings.json | 35 + .../.claude/skills/check-vacuum/SKILL.md | 15 + .../skills/compare-hyperparameters/SKILL.md | 18 + .../.claude/skills/verify-results/SKILL.md | 14 + q-learning-9b/.gitignore | 4 + q-learning-9b/CLAUDE.md | 65 + q-learning-9b/README.md | 215 ++ q-learning-9b/aima/LICENSE | 9 + q-learning-9b/aima/UPSTREAM.md | 10 + q-learning-9b/aima/__init__.py | 1 + q-learning-9b/aima/agents.py | 1120 ++++++++ q-learning-9b/aima/csp.py | 1502 ++++++++++ q-learning-9b/aima/deep_learning.py | 658 +++++ q-learning-9b/aima/game_theory.py | 257 ++ q-learning-9b/aima/games.py | 672 +++++ q-learning-9b/aima/ipyviews.py | 175 ++ q-learning-9b/aima/knowledge.py | 431 +++ q-learning-9b/aima/learning.py | 1437 ++++++++++ q-learning-9b/aima/logic.py | 2286 +++++++++++++++ q-learning-9b/aima/making_simple_decisions.py | 11 + q-learning-9b/aima/mdp.py | 593 ++++ q-learning-9b/aima/nlp.py | 721 +++++ q-learning-9b/aima/notebook_utils.py | 1311 +++++++++ q-learning-9b/aima/perception.py | 517 ++++ q-learning-9b/aima/planning.py | 2524 +++++++++++++++++ q-learning-9b/aima/probabilistic_learning.py | 158 ++ q-learning-9b/aima/probability.py | 1389 +++++++++ q-learning-9b/aima/reinforcement_learning.py | 376 +++ q-learning-9b/aima/search.py | 2067 ++++++++++++++ q-learning-9b/aima/text.py | 435 +++ q-learning-9b/aima/utils.py | 872 ++++++ q-learning-9b/context/environment.md | 65 + q-learning-9b/context/requirements.md | 96 + q-learning-9b/learning_todos.py | 135 + q-learning-9b/q_learning.ipynb | 533 ++++ q-learning-9b/q_learning_demo.py | 72 + q-learning-9b/requirements.txt | 4 + q-learning-9b/results.json | 55 + q-learning-9b/scripts/check_project.py | 148 + q-learning-9b/scripts/guard_tools.py | 82 + .../scripts/sweep_hyperparameters.py | 93 + q-learning-9b/scripts/test_code_features.py | 93 + q-learning-9b/scripts/test_guard_tools.py | 68 + q-learning-9b/scripts/test_q_learning.py | 317 +++ q-learning-9b/scripts/verify_results.py | 119 + tests/test_mini_coding_agent.py | 70 + 54 files changed, 22082 insertions(+), 20 deletions(-) create mode 100644 q-learning-9b/.claude/agents/configuration-reviewer.md create mode 100644 q-learning-9b/.claude/agents/environment-specialist.md create mode 100644 q-learning-9b/.claude/agents/requirements-specialist.md create mode 100644 q-learning-9b/.claude/settings.json create mode 100644 q-learning-9b/.claude/skills/check-vacuum/SKILL.md create mode 100644 q-learning-9b/.claude/skills/compare-hyperparameters/SKILL.md create mode 100644 q-learning-9b/.claude/skills/verify-results/SKILL.md create mode 100644 q-learning-9b/.gitignore create mode 100644 q-learning-9b/CLAUDE.md create mode 100644 q-learning-9b/README.md create mode 100644 q-learning-9b/aima/LICENSE create mode 100644 q-learning-9b/aima/UPSTREAM.md create mode 100644 q-learning-9b/aima/__init__.py create mode 100644 q-learning-9b/aima/agents.py create mode 100644 q-learning-9b/aima/csp.py create mode 100644 q-learning-9b/aima/deep_learning.py create mode 100644 q-learning-9b/aima/game_theory.py create mode 100644 q-learning-9b/aima/games.py create mode 100644 q-learning-9b/aima/ipyviews.py create mode 100644 q-learning-9b/aima/knowledge.py create mode 100644 q-learning-9b/aima/learning.py create mode 100644 q-learning-9b/aima/logic.py create mode 100644 q-learning-9b/aima/making_simple_decisions.py create mode 100644 q-learning-9b/aima/mdp.py create mode 100644 q-learning-9b/aima/nlp.py create mode 100644 q-learning-9b/aima/notebook_utils.py create mode 100644 q-learning-9b/aima/perception.py create mode 100644 q-learning-9b/aima/planning.py create mode 100644 q-learning-9b/aima/probabilistic_learning.py create mode 100644 q-learning-9b/aima/probability.py create mode 100644 q-learning-9b/aima/reinforcement_learning.py create mode 100644 q-learning-9b/aima/search.py create mode 100644 q-learning-9b/aima/text.py create mode 100644 q-learning-9b/aima/utils.py create mode 100644 q-learning-9b/context/environment.md create mode 100644 q-learning-9b/context/requirements.md create mode 100644 q-learning-9b/learning_todos.py create mode 100644 q-learning-9b/q_learning.ipynb create mode 100644 q-learning-9b/q_learning_demo.py create mode 100644 q-learning-9b/requirements.txt create mode 100644 q-learning-9b/results.json create mode 100644 q-learning-9b/scripts/check_project.py create mode 100644 q-learning-9b/scripts/guard_tools.py create mode 100644 q-learning-9b/scripts/sweep_hyperparameters.py create mode 100644 q-learning-9b/scripts/test_code_features.py create mode 100644 q-learning-9b/scripts/test_guard_tools.py create mode 100644 q-learning-9b/scripts/test_q_learning.py create mode 100644 q-learning-9b/scripts/verify_results.py diff --git a/README.md b/README.md index 78c1767..ee093bf 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ The model backend is currently based on Ollama.   -## Six Core Components +## Seven Components Six core components of a coding agent -This coding harness is organized around six practical building blocks: +This coding harness is organized around seven components: 1. **Live repo context** The agent collects stable workspace facts upfront, such as repo layout, instructions, and git state. @@ -47,6 +47,90 @@ This coding harness is organized around six practical building blocks: The runtime keeps both a full durable transcript and a smaller working memory so sessions can be resumed while preserving important state via working memory. 6. **Delegation and bounded subagents** Scoped subtasks can be delegated to helper agents that inherit enough context to help (but operate within limits). +7. **Agent skills** discover skill folders, select an applicable skill, add its instructions to the model prompt, and log the decision. + +## CS440 Project 2 + +| Path | Purpose | +| --- | --- | +| `mini_coding_agent.py` | Agent loop, skill discovery and matching, `use_skill` tool, and logs | +| `.claude/skills/` | Original Lab 5 skills | +| `intent/` | SDLC intent, specification, and implementation plan | +| `q-learning-9b/` | Vacuum robot demo, notebook, tests, and saved results | +| `tests/` | Agent and skill regression tests | + +The agent searches the target repository root for one-level skill folders under +`skills/`, `skill/`, and `.claude/skills/`. Each folder contains `SKILL.md` +or `skill.md`. The registry reads the name, description, and optional keywords. +It scores the prompt against each skill's metadata using word-count cosine +similarity and selects the best score at or above 0.25. A `/skill-name` prompt +or a model call to `use_skill` selects by exact name. The skill body is +included in subsequent model prompts for that turn. The model follows it with +the agent's existing tools and approval rules. + +### Vacuum robot demonstration + +The `q-learning-9b/` folder inside this repository contains a copy of CS440 Lab 5 +with its original `check-vacuum` skill and simulator. The demo workspace contains +`learning_todos.py`, `q_learning.ipynb`, and `q_learning_demo.py` alongside +the simulator and tests. From the repository root, install the demo dependencies +and verify the included implementation: + +```powershell +python -m pip install -r q-learning-9b/requirements.txt +Push-Location .\q-learning-9b +python q_learning_demo.py verify +Pop-Location +``` + +To open the included demo workspace in the agent, start Ollama per the +[requirements](#requirements) and run this from the repository root: + +```powershell +python mini_coding_agent.py --cwd q-learning-9b --approval ask --max-steps 15 --max-new-tokens 2048 +``` + +To start from TODOs, run `python q_learning_demo.py prepare` inside a fresh +disposable demo checkout that has the notebook and helper but no +`learning_todos.py`. Preparation generates the learner starter, clears +notebook outputs, and removes any existing `results.json` from that checkout. +Launch the agent from the repository root and pass that checkout's path to +`--cwd`. At the agent prompt, enter `/skills` to verify discovery. Then try: + +> Check the Lab 5 notebook for the vacuum robot and use focused specialists to review the environment, requirements, and code features. To solve the Lab 4 vacuum robot problem, implement the five Q-learning TODOs in learning_todos.py. Read the environment and tests for the expected interfaces. Use only observations from the simulator. Then run python q_learning_demo.py verify and report the results. Keep the simulator and tests intact. + +Review the tool arguments when the agent requests approval. Approve the +checker command, learner-file edits, and verification command if they match the +task. The verification command copies the learner into the notebook, +runs the checker and Q-learning tests, then trains and evaluates if those +checks succeed. A syntax checker pass can still contain unfinished TODOs. +The `verify` command writes fresh `results.json` only after successful training +and evaluation. To trigger the skill explicitly, prefix the prompt with +`/check-vacuum`. + +To display the skill log from a run in the included demo workspace, open a +second terminal at the repository root: + +```powershell +Get-Content -LiteralPath '.\q-learning-9b\.mini-coding-agent\skills.log' | + ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object { $_.depth -eq 0 } | + ConvertTo-Json -Depth 6 +``` + +Show the `prompt`, `similarity_check`, and `skill_executed` records for the +same session. The execution record means the instructions were activated. +Use the tool calls, the source diff, and the test or training output to show +what the agent completed and what remains. +Logs store prompt text locally. Delegated calls have their own session IDs and +`depth: 1` records. + +To run the automated tests from the agent repository: + +```powershell +python -m pip install -e . pytest +python -m pytest -q +```   ## Requirements diff --git a/intent/intent.md b/intent/intent.md index 9b37e3e..51c8ae3 100644 --- a/intent/intent.md +++ b/intent/intent.md @@ -1,9 +1,12 @@ # Coding Agent Skills ## Repository -This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. +This project extends the upstream lightweight coding agent with a seventh component for skills. ## Objective The code agent must be able to identify a folder skill (such as .claude\skills\pr-review) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. Each skill has a description which the agent will use to identify the right skill to use. The new component must log the prompt, the similarity check, and the executed skill. + +Demonstrate the component by attempting the CS440 Lab 4 vacuum robot problem +with the original Lab 5 `check-vacuum` skill and a Q-learning starter. diff --git a/intent/plan.md b/intent/plan.md index 3619ba9..e2a721a 100644 --- a/intent/plan.md +++ b/intent/plan.md @@ -1,5 +1,19 @@ # Plan: Agent Skills for `mini_coding_agent.py` +## Implementation status + +The implementation uses the existing agent loop. It searches `skills/`, +`skill/`, and `.claude/skills/`, uses word-count cosine similarity with a +0.25 threshold, and supports explicit selection by name from both prompts +and the `use_skill` tool. It appends JSON Lines to +`.mini-coding-agent/skills.log`. The design proposal below records the +original plan. The README documents the implemented behavior and demo steps. + +For the demonstration, prepare a disposable Lab 5 checkout with five +Q-learning TODOs. Use the original `check-vacuum` skill to guide the agent's +attempt on the vacuum robot problem. Verify the prompt, selection, and +activation records, then inspect the actual edits and verification output. + Derived from `spec.md` and a read of `mini_coding_agent.py` (1,019 lines, stdlib-only, Ollama backend, `MiniAgent` with six numbered components). ## 1. Goal @@ -194,4 +208,4 @@ Use the existing `FakeModelClient` for deterministic agent-loop tests, with a `t 2. Phase 4.1-4.7 (prompt trigger path) and its e2e test. 3. Phase 4.3-4.4, 4.8 (`use_skill` tool path) and its e2e test. 4. Phase 4.9-4.11 (session, delegation, `build_agent`). -5. Phase 5 (CLI), then Phase 7 (docs), then Phase 6.10 threshold tuning against real prompts with Ollama. \ No newline at end of file +5. Phase 5 (CLI), then Phase 7 (docs), then Phase 6.10 threshold tuning against real prompts with Ollama. diff --git a/intent/spec.md b/intent/spec.md index b4ede10..2c1fd86 100644 --- a/intent/spec.md +++ b/intent/spec.md @@ -1,7 +1,7 @@ # Coding Agent Skills Specification ## Repository -This repository contains code which implements a lightweight coding agent. The agent does not yet have skill functionality. +The upstream lightweight coding agent provides the runtime for this component. ## Objective The code agent must be able to identify a folder skill (such as `.claude\skills\pr-review`) and trigger a skill.md file by executing the instructions when an appropriate prompt is triggered or when the code agent needs a specific agent skill during execution. @@ -20,3 +20,12 @@ The new component must log the prompt, the similarity check, and the executed sk - Skills should be organized in folder structures within `.claude\skills\` - Each skill folder contains a `skill.md` file with the skill's description and instructions - The logging mechanism should be integrated into the agent's execution flow + +## Acceptance checks + +- A matching ordinary prompt loads the skill body into the current model prompt. +- A model request for an installed skill can activate it during a turn. +- An unrelated subsequent prompt does not inherit the previously active skill. +- The log can connect a trigger, selection result, and activation by session ID. +- The demo attempts the Lab 4 vacuum robot problem using the original Lab 5 + skill and reports the actual edits, tests, and training results it obtains. diff --git a/mini_coding_agent.py b/mini_coding_agent.py index 63e0ca8..88bcff3 100644 --- a/mini_coding_agent.py +++ b/mini_coding_agent.py @@ -325,7 +325,7 @@ def match(self, prompt): """Return (best_skill_or_None, scores, trigger). An explicit /skill-name always wins.""" lowered = str(prompt).lower() for name, skill in self.skills.items(): - if f"/{name.lower()}" in lowered: + if re.search(rf"(?'.format(getattr(self, '__name__', self.__class__.__name__)) + + def is_alive(self): + """Things that are 'alive' should return true.""" + return hasattr(self, 'alive') and self.alive + + def show_state(self): + """Display the agent's internal state. Subclasses should override.""" + print("I don't know how to show_state.") + + def display(self, canvas, x, y, width, height): + """Display an image of this Thing on the canvas.""" + # Do we need this? + pass + + +class Agent(Thing): + """An Agent is a subclass of Thing with one required instance attribute + (aka slot), .program, which should hold a function that takes one argument, + the percept, and returns an action. (What counts as a percept or action + will depend on the specific environment in which the agent exists.) + Note that 'program' is a slot, not a method. If it were a method, then the + program could 'cheat' and look at aspects of the agent. It's not supposed + to do that: the program can only look at the percepts. An agent program + that needs a model of the world (and of the agent itself) will have to + build and maintain its own model. There is an optional slot, .performance, + which is a number giving the performance measure of the agent in its + environment.""" + + def __init__(self, program=None): + self.alive = True + self.bump = False + self.holding = [] + self.performance = 0 + if program is None or not isinstance(program, collections.abc.Callable): + print("Can't find a valid program for {}, falling back to default.".format(self.__class__.__name__)) + + def program(percept): + return eval(input('Percept={}; action? '.format(percept))) + + self.program = program + + def can_grab(self, thing): + """Return True if this agent can grab this thing. + Override for appropriate subclasses of Agent and Thing.""" + return False + + +def TraceAgent(agent): + """Wrap the agent's program to print its input and output. This will let + you see what the agent is doing in the environment.""" + old_program = agent.program + + def new_program(percept): + action = old_program(percept) + print('{} perceives {} and does {}'.format(agent, percept, action)) + return action + + agent.program = new_program + return agent + + +# ______________________________________________________________________________ + + +def TableDrivenAgentProgram(table): + """ + [Figure 2.7] + This agent selects an action based on the percept sequence. + It is practical only for tiny domains. + To customize it, provide as table a dictionary of all + {percept_sequence:action} pairs. + """ + percepts = [] + + def program(percept): + percepts.append(percept) + action = table.get(tuple(percepts)) + return action + + return program + + +def RandomAgentProgram(actions): + """An agent that chooses an action at random, ignoring all percepts. + >>> list = ['Right', 'Left', 'Suck', 'NoOp'] + >>> program = RandomAgentProgram(list) + >>> agent = Agent(program) + >>> environment = TrivialVacuumEnvironment() + >>> environment.add_thing(agent) + >>> environment.run() + >>> environment.status == {(1, 0): 'Clean' , (0, 0): 'Clean'} + True + """ + return lambda percept: random.choice(actions) + + +# ______________________________________________________________________________ + + +def SimpleReflexAgentProgram(rules, interpret_input): + """ + [Figure 2.10] + This agent takes action based solely on the percept. + """ + + def program(percept): + state = interpret_input(percept) + rule = rule_match(state, rules) + action = rule.action + return action + + return program + + +def ModelBasedReflexAgentProgram(rules, update_state, model): + """ + [Figure 2.12] + This agent takes action based on the percept and state. + """ + + def program(percept): + program.state = update_state(program.state, program.action, percept, model) + rule = rule_match(program.state, rules) + action = rule.action + return action + + program.state = program.action = None + return program + + +def rule_match(state, rules): + """Find the first rule that matches state.""" + for rule in rules: + if rule.matches(state): + return rule + + +# ______________________________________________________________________________ + + +loc_A, loc_B = (0, 0), (1, 0) # The two locations for the Vacuum world + + +def RandomVacuumAgent(): + """Randomly choose one of the actions from the vacuum environment. + >>> agent = RandomVacuumAgent() + >>> environment = TrivialVacuumEnvironment() + >>> environment.add_thing(agent) + >>> environment.run() + >>> environment.status == {(1,0):'Clean' , (0,0) : 'Clean'} + True + """ + return Agent(RandomAgentProgram(['Right', 'Left', 'Suck', 'NoOp'])) + + +def TableDrivenVacuumAgent(): + """Tabular approach towards vacuum world as mentioned in [Figure 2.3] + >>> agent = TableDrivenVacuumAgent() + >>> environment = TrivialVacuumEnvironment() + >>> environment.add_thing(agent) + >>> environment.run() + >>> environment.status == {(1,0):'Clean' , (0,0) : 'Clean'} + True + """ + table = {((loc_A, 'Clean'),): 'Right', + ((loc_A, 'Dirty'),): 'Suck', + ((loc_B, 'Clean'),): 'Left', + ((loc_B, 'Dirty'),): 'Suck', + ((loc_A, 'Dirty'), (loc_A, 'Clean')): 'Right', + ((loc_A, 'Clean'), (loc_B, 'Dirty')): 'Suck', + ((loc_B, 'Clean'), (loc_A, 'Dirty')): 'Suck', + ((loc_B, 'Dirty'), (loc_B, 'Clean')): 'Left', + ((loc_A, 'Dirty'), (loc_A, 'Clean'), (loc_B, 'Dirty')): 'Suck', + ((loc_B, 'Dirty'), (loc_B, 'Clean'), (loc_A, 'Dirty')): 'Suck'} + return Agent(TableDrivenAgentProgram(table)) + + +def ReflexVacuumAgent(): + """ + [Figure 2.8] + A reflex agent for the two-state vacuum environment. + >>> agent = ReflexVacuumAgent() + >>> environment = TrivialVacuumEnvironment() + >>> environment.add_thing(agent) + >>> environment.run() + >>> environment.status == {(1,0):'Clean' , (0,0) : 'Clean'} + True + """ + + def program(percept): + location, status = percept + if status == 'Dirty': + return 'Suck' + elif location == loc_A: + return 'Right' + elif location == loc_B: + return 'Left' + + return Agent(program) + + +def ModelBasedVacuumAgent(): + """An agent that keeps track of what locations are clean or dirty. + >>> agent = ModelBasedVacuumAgent() + >>> environment = TrivialVacuumEnvironment() + >>> environment.add_thing(agent) + >>> environment.run() + >>> environment.status == {(1,0):'Clean' , (0,0) : 'Clean'} + True + """ + model = {loc_A: None, loc_B: None} + + def program(percept): + """Same as ReflexVacuumAgent, except if everything is clean, do NoOp.""" + location, status = percept + model[location] = status # Update the model here + if model[loc_A] == model[loc_B] == 'Clean': + return 'NoOp' + elif status == 'Dirty': + return 'Suck' + elif location == loc_A: + return 'Right' + elif location == loc_B: + return 'Left' + + return Agent(program) + + +# ______________________________________________________________________________ + + +class Environment: + """Abstract class representing an Environment. 'Real' Environment classes + inherit from this. Your Environment will typically need to implement:: + + percept: Define the percept that an agent sees. + execute_action: Define the effects of executing an action; + also update the agent.performance slot. + + The environment keeps a list of .things and .agents (which is a subset + of .things). Each agent has a .performance slot, initialized to 0. + Each thing has a .location slot, even though some environments may not + need this.""" + + def __init__(self): + self.things = [] + self.agents = [] + + def thing_classes(self): + """Return the list of Thing subclasses that may appear in this environment.""" + return [] # List of classes that can go into environment + + def percept(self, agent): + """Return the percept that the agent sees at this point. (Implement this.)""" + raise NotImplementedError + + def execute_action(self, agent, action): + """Change the world to reflect this action. (Implement this.)""" + raise NotImplementedError + + def default_location(self, thing): + """Default location to place a new thing with unspecified location.""" + return None + + def exogenous_change(self): + """If there is spontaneous change in the world, override this.""" + pass + + def is_done(self): + """By default, we're done when we can't find a live agent.""" + return not any(agent.is_alive() for agent in self.agents) + + def step(self): + """Run the environment for one time step. If the + actions and exogenous changes are independent, this method will + do. If there are interactions between them, you'll need to + override this method.""" + if not self.is_done(): + actions = [] + for agent in self.agents: + if agent.alive: + actions.append(agent.program(self.percept(agent))) + else: + actions.append("") + for (agent, action) in zip(self.agents, actions): + self.execute_action(agent, action) + self.exogenous_change() + + def run(self, steps=1000): + """Run the Environment for given number of time steps.""" + for step in range(steps): + if self.is_done(): + return + self.step() + + def list_things_at(self, location, tclass=Thing): + """Return all things exactly at a given location.""" + if isinstance(location, numbers.Number): + return [thing for thing in self.things + if thing.location == location and isinstance(thing, tclass)] + return [thing for thing in self.things + if all(x == y for x, y in zip(thing.location, location)) and isinstance(thing, tclass)] + + def some_things_at(self, location, tclass=Thing): + """Return true if at least one of the things at location + is an instance of class tclass (or a subclass).""" + return self.list_things_at(location, tclass) != [] + + def add_thing(self, thing, location=None): + """Add a thing to the environment, setting its location. For + convenience, if thing is an agent program we make a new agent + for it. (Shouldn't need to override this.)""" + if not isinstance(thing, Thing): + thing = Agent(thing) + if thing in self.things: + print("Can't add the same thing twice") + else: + thing.location = location if location is not None else self.default_location(thing) + self.things.append(thing) + if isinstance(thing, Agent): + thing.performance = 0 + self.agents.append(thing) + + def delete_thing(self, thing): + """Remove a thing from the environment.""" + try: + self.things.remove(thing) + except ValueError as e: + print(e) + print(" in Environment delete_thing") + print(" Thing to be removed: {} at {}".format(thing, thing.location)) + print(" from list: {}".format([(thing, thing.location) for thing in self.things])) + if thing in self.agents: + self.agents.remove(thing) + + +class Direction: + """A direction class for agents that want to move in a 2D plane. + + Usage:: + + d = Direction("down") + # to change directions: + d = d + "right" or d = d + Direction.R # both do the same thing + + Note that the argument to __add__ must be a string and not a Direction + object, and it can only be 'right' or 'left'.""" + + R = "right" + L = "left" + U = "up" + D = "down" + + def __init__(self, direction): + self.direction = direction + + def __add__(self, heading): + """ + >>> d = Direction('right') + >>> l1 = d.__add__(Direction.L) + >>> l2 = d.__add__(Direction.R) + >>> l1.direction + 'up' + >>> l2.direction + 'down' + >>> d = Direction('down') + >>> l1 = d.__add__('right') + >>> l2 = d.__add__('left') + >>> l1.direction == Direction.L + True + >>> l2.direction == Direction.R + True + """ + if self.direction == self.R: + return { + self.R: Direction(self.D), + self.L: Direction(self.U), + }.get(heading, None) + elif self.direction == self.L: + return { + self.R: Direction(self.U), + self.L: Direction(self.D), + }.get(heading, None) + elif self.direction == self.U: + return { + self.R: Direction(self.R), + self.L: Direction(self.L), + }.get(heading, None) + elif self.direction == self.D: + return { + self.R: Direction(self.L), + self.L: Direction(self.R), + }.get(heading, None) + + def move_forward(self, from_location): + """ + >>> d = Direction('up') + >>> l1 = d.move_forward((0, 0)) + >>> l1 + (0, -1) + >>> d = Direction(Direction.R) + >>> l1 = d.move_forward((0, 0)) + >>> l1 + (1, 0) + """ + # get the iterable class to return + iclass = from_location.__class__ + x, y = from_location + if self.direction == self.R: + return iclass((x + 1, y)) + elif self.direction == self.L: + return iclass((x - 1, y)) + elif self.direction == self.U: + return iclass((x, y - 1)) + elif self.direction == self.D: + return iclass((x, y + 1)) + + +class XYEnvironment(Environment): + """This class is for environments on a 2D plane, with locations + labelled by (x, y) points, either discrete or continuous. + + Agents perceive things within a radius. Each agent in the + environment has a .location slot which should be a location such + as (0, 1), and a .holding slot, which should be a list of things + that are held.""" + + def __init__(self, width=10, height=10): + super().__init__() + + self.width = width + self.height = height + self.observers = [] + # Sets iteration start and end (no walls). + self.x_start, self.y_start = (0, 0) + self.x_end, self.y_end = (self.width - 1, self.height - 1) + + perceptible_distance = 1 + + def things_near(self, location, radius=None): + """Return all things within radius of location.""" + if radius is None: + radius = self.perceptible_distance + radius2 = radius * radius + return [(thing, radius2 - distance_squared(location, thing.location)) + for thing in self.things if distance_squared( + location, thing.location) <= radius2] + + def percept(self, agent): + """By default, agent perceives things within a default radius.""" + return self.things_near(agent.location) + + def execute_action(self, agent, action): + """Apply a motion or manipulation action for the agent. Supports turning + ('TurnRight'/'TurnLeft'), moving one step ('Forward', setting agent.bump on + a collision), grabbing a grabbable thing at the agent's location ('Grab'), + and dropping the last held thing ('Release').""" + agent.bump = False + if action == 'TurnRight': + agent.direction += Direction.R + elif action == 'TurnLeft': + agent.direction += Direction.L + elif action == 'Forward': + agent.bump = self.move_to(agent, agent.direction.move_forward(agent.location)) + elif action == 'Grab': + things = [thing for thing in self.list_things_at(agent.location) if agent.can_grab(thing)] + if things: + agent.holding.append(things[0]) + print("Grabbing ", things[0].__class__.__name__) + self.delete_thing(things[0]) + elif action == 'Release': + if agent.holding: + dropped = agent.holding.pop() + print("Dropping ", dropped.__class__.__name__) + self.add_thing(dropped, location=agent.location) + + def default_location(self, thing): + """Return a random inbounds location that contains no Obstacle.""" + location = self.random_location_inbounds() + while self.some_things_at(location, Obstacle): + # we will find a random location with no obstacles + location = self.random_location_inbounds() + return location + + def move_to(self, thing, destination): + """Move a thing to a new location. Returns True on success or False if there is an Obstacle. + If thing is holding anything, they move with him.""" + thing.bump = self.some_things_at(destination, Obstacle) + if not thing.bump: + thing.location = destination + for o in self.observers: + o.thing_moved(thing) + for t in thing.holding: + self.delete_thing(t) + self.add_thing(t, destination) + t.location = destination + return thing.bump + + def add_thing(self, thing, location=None, exclude_duplicate_class_items=False): + """Add things to the world. If (exclude_duplicate_class_items) then the item won't be + added if the location has at least one item of the same class.""" + if location is None: + super().add_thing(thing) + elif self.is_inbounds(location): + if (exclude_duplicate_class_items and + any(isinstance(t, thing.__class__) for t in self.list_things_at(location))): + return + super().add_thing(thing, location) + + def is_inbounds(self, location): + """Checks to make sure that the location is inbounds (within walls if we have walls)""" + x, y = location + return not (x < self.x_start or x > self.x_end or y < self.y_start or y > self.y_end) + + def random_location_inbounds(self, exclude=None): + """Returns a random location that is inbounds (within walls if we have walls)""" + location = (random.randint(self.x_start, self.x_end), + random.randint(self.y_start, self.y_end)) + if exclude is not None: + while location == exclude: + location = (random.randint(self.x_start, self.x_end), + random.randint(self.y_start, self.y_end)) + return location + + def delete_thing(self, thing): + """Deletes thing, and everything it is holding (if thing is an agent)""" + if isinstance(thing, Agent): + del thing.holding + + super().delete_thing(thing) + for obs in self.observers: + obs.thing_deleted(thing) + + def add_walls(self): + """Put walls around the entire perimeter of the grid.""" + for x in range(self.width): + self.add_thing(Wall(), (x, 0)) + self.add_thing(Wall(), (x, self.height - 1)) + for y in range(1, self.height - 1): + self.add_thing(Wall(), (0, y)) + self.add_thing(Wall(), (self.width - 1, y)) + + # Updates iteration start and end (with walls). + self.x_start, self.y_start = (1, 1) + self.x_end, self.y_end = (self.width - 1, self.height - 1) + + def add_observer(self, observer): + """Adds an observer to the list of observers. + An observer is typically an EnvGUI. + + Each observer is notified of changes in move_to and add_thing, + by calling the observer's methods thing_moved(thing) + and thing_added(thing, loc).""" + self.observers.append(observer) + + def turn_heading(self, heading, inc): + """Return the heading to the left (inc=+1) or right (inc=-1) of heading.""" + return turn_heading(heading, inc) + + +class Obstacle(Thing): + """Something that can cause a bump, preventing an agent from + moving into the same square it's in.""" + pass + + +class Wall(Obstacle): + """An impassable Obstacle forming the boundary or interior walls of a grid.""" + pass + + +# ______________________________________________________________________________ + + +class GraphicEnvironment(XYEnvironment): + """An XYEnvironment that visualises itself in a Jupyter notebook using an + ipythonblocks BlockGrid, colouring each cell according to the class of the + last thing placed there.""" + + def __init__(self, width=10, height=10, boundary=True, color={}, display=False): + """Define all the usual XYEnvironment characteristics, + but initialise a BlockGrid for GUI too.""" + super().__init__(width, height) + # imported lazily so the rest of aima.agents (and modules that import it, + # e.g. logic/csp) can be used without the optional ipythonblocks GUI dep + try: + from ipythonblocks import BlockGrid + except ImportError as e: + raise ImportError("GraphicEnvironment needs the optional 'ipythonblocks' " + "package; install it with `pip install ipythonblocks`.") from e + self.grid = BlockGrid(width, height, fill=(200, 200, 200)) + if display: + self.grid.show() + self.visible = True + else: + self.visible = False + self.bounded = boundary + self.colors = color + + def get_world(self): + """Returns all the items in the world in a format + understandable by the ipythonblocks BlockGrid.""" + result = [] + x_start, y_start = (0, 0) + x_end, y_end = self.width, self.height + for x in range(x_start, x_end): + row = [] + for y in range(y_start, y_end): + row.append(self.list_things_at((x, y))) + result.append(row) + return result + + """ + def run(self, steps=1000, delay=1): + "" "Run the Environment for given number of time steps, + but update the GUI too." "" + for step in range(steps): + sleep(delay) + if self.visible: + self.reveal() + if self.is_done(): + if self.visible: + self.reveal() + return + self.step() + if self.visible: + self.reveal() + """ + + def run(self, steps=1000, delay=1): + """Run the Environment for given number of time steps, + but update the GUI too.""" + for step in range(steps): + self.update(delay) + if self.is_done(): + break + self.step() + self.update(delay) + + def update(self, delay=1): + """Pause for delay seconds, then redraw the world in the GUI.""" + sleep(delay) + self.reveal() + + def reveal(self): + """Display the BlockGrid for this world - the last thing to be added + at a location defines the location color.""" + self.draw_world() + # wait for the world to update and + # apply changes to the same grid instead + # of making a new one. + clear_output(1) + self.grid.show() + self.visible = True + + def draw_world(self): + """Repaint the BlockGrid, colouring each occupied cell by the class of the + last thing at that location using the configured color mapping.""" + self.grid[:] = (200, 200, 200) + world = self.get_world() + for x in range(0, len(world)): + for y in range(0, len(world[x])): + if len(world[x][y]): + self.grid[y, x] = self.colors[world[x][y][-1].__class__.__name__] + + def conceal(self): + """Hide the BlockGrid for this world""" + self.visible = False + display(HTML('')) + + +# ______________________________________________________________________________ +# Continuous environment + +class ContinuousWorld(Environment): + """Model for Continuous World""" + + def __init__(self, width=10, height=10): + super().__init__() + self.width = width + self.height = height + + def add_obstacle(self, coordinates): + """Add a PolygonObstacle defined by the given list of vertex coordinates.""" + self.things.append(PolygonObstacle(coordinates)) + + +class PolygonObstacle(Obstacle): + """An Obstacle in a ContinuousWorld whose shape is a polygon described by a + list of vertex coordinates.""" + + def __init__(self, coordinates): + """Coordinates is a list of tuples.""" + super().__init__() + self.coordinates = coordinates + + +# ______________________________________________________________________________ +# Vacuum environment + + +class Dirt(Thing): + """A piece of dirt that a vacuum agent can clean up.""" + pass + + +class VacuumEnvironment(XYEnvironment): + """The environment of [Ex. 2.12]. Agent perceives dirty or clean, + and bump (into obstacle) or not; 2D discrete world of unknown size; + performance measure is 100 for each dirt cleaned, and -1 for + each turn taken.""" + + def __init__(self, width=10, height=10): + super().__init__(width, height) + self.add_walls() + + def thing_classes(self): + """Return the Thing/Agent classes that may populate the vacuum world.""" + return [Wall, Dirt, ReflexVacuumAgent, RandomVacuumAgent, + TableDrivenVacuumAgent, ModelBasedVacuumAgent] + + def percept(self, agent): + """The percept is a tuple of ('Dirty' or 'Clean', 'Bump' or 'None'). + Unlike the TrivialVacuumEnvironment, location is NOT perceived.""" + status = ('Dirty' if self.some_things_at( + agent.location, Dirt) else 'Clean') + bump = ('Bump' if agent.bump else 'None') + return status, bump + + def execute_action(self, agent, action): + """Carry out the agent's action. 'Suck' removes dirt at the agent's location + and adds 100 to its performance; movement actions are delegated to the + XYEnvironment, and every action other than 'NoOp' costs 1 performance point.""" + agent.bump = False + if action == 'Suck': + dirt_list = self.list_things_at(agent.location, Dirt) + if dirt_list != []: + dirt = dirt_list[0] + agent.performance += 100 + self.delete_thing(dirt) + else: + super().execute_action(agent, action) + + if action != 'NoOp': + agent.performance -= 1 + + +class TrivialVacuumEnvironment(Environment): + """This environment has two locations, A and B. Each can be Dirty + or Clean. The agent perceives its location and the location's + status. This serves as an example of how to implement a simple + Environment.""" + + def __init__(self): + super().__init__() + self.status = {loc_A: random.choice(['Clean', 'Dirty']), + loc_B: random.choice(['Clean', 'Dirty'])} + + def thing_classes(self): + """Return the Thing/Agent classes that may populate this vacuum world.""" + return [Wall, Dirt, ReflexVacuumAgent, RandomVacuumAgent, TableDrivenVacuumAgent, ModelBasedVacuumAgent] + + def percept(self, agent): + """Returns the agent's location, and the location status (Dirty/Clean).""" + return agent.location, self.status[agent.location] + + def execute_action(self, agent, action): + """Change agent's location and/or location's status; track performance. + Score 10 for each dirt cleaned; -1 for each move.""" + if action == 'Right': + agent.location = loc_B + agent.performance -= 1 + elif action == 'Left': + agent.location = loc_A + agent.performance -= 1 + elif action == 'Suck': + if self.status[agent.location] == 'Dirty': + agent.performance += 10 + self.status[agent.location] = 'Clean' + + def default_location(self, thing): + """Agents start in either location at random.""" + return random.choice([loc_A, loc_B]) + + +# ______________________________________________________________________________ +# The Wumpus World + + +class Gold(Thing): + """The gold the explorer is trying to grab in the Wumpus World.""" + + def __eq__(self, rhs): + """All Gold are equal""" + return rhs.__class__ == Gold + + pass + + +class Bump(Thing): + """Percept signalling that the explorer walked into a wall.""" + pass + + +class Glitter(Thing): + """Percept indicating that gold is in the explorer's current room.""" + pass + + +class Pit(Thing): + """A pit that kills the explorer if entered.""" + pass + + +class Breeze(Thing): + """Percept felt in rooms adjacent to a Pit.""" + pass + + +class Arrow(Thing): + """The single arrow the explorer can shoot to try to kill the wumpus.""" + pass + + +class Scream(Thing): + """Percept heard throughout the world when the wumpus is killed.""" + pass + + +class Wumpus(Agent): + """The wumpus: a monster that kills the explorer sharing its room.""" + screamed = False + pass + + +class Stench(Thing): + """Percept smelled in rooms adjacent to the Wumpus.""" + pass + + +class Explorer(Agent): + """The agent that explores the Wumpus World, seeking gold while avoiding + pits and the wumpus.""" + holding = [] + has_arrow = True + killed_by = "" + direction = Direction("right") + + def can_grab(self, thing): + """Explorer can only grab gold""" + return thing.__class__ == Gold + + +class WumpusEnvironment(XYEnvironment): + """A grid implementation of the Wumpus World from Chapter 7: a cave of rooms + surrounded by walls, holding pits, the wumpus and gold, in which an Explorer + agent perceives stench, breeze, glitter, bump and scream.""" + + pit_probability = 0.2 # Probability to spawn a pit in a location. (From Chapter 7.2) + + # Room should be 4x4 grid of rooms. The extra 2 for walls + + def __init__(self, agent_program, width=6, height=6): + super().__init__(width, height) + self.init_world(agent_program) + + def init_world(self, program): + """Spawn items in the world based on probabilities from the book""" + + "WALLS" + self.add_walls() + + "PITS" + for x in range(self.x_start, self.x_end): + for y in range(self.y_start, self.y_end): + if random.random() < self.pit_probability: + self.add_thing(Pit(), (x, y), True) + self.add_thing(Breeze(), (x - 1, y), True) + self.add_thing(Breeze(), (x, y - 1), True) + self.add_thing(Breeze(), (x + 1, y), True) + self.add_thing(Breeze(), (x, y + 1), True) + + "WUMPUS" + w_x, w_y = self.random_location_inbounds(exclude=(1, 1)) + self.add_thing(Wumpus(lambda x: ""), (w_x, w_y), True) + self.add_thing(Stench(), (w_x - 1, w_y), True) + self.add_thing(Stench(), (w_x + 1, w_y), True) + self.add_thing(Stench(), (w_x, w_y - 1), True) + self.add_thing(Stench(), (w_x, w_y + 1), True) + + "GOLD" + self.add_thing(Gold(), self.random_location_inbounds(exclude=(1, 1)), True) + + "AGENT" + self.add_thing(Explorer(program), (1, 1), True) + + def get_world(self, show_walls=True): + """Return the items in the world""" + result = [] + x_start, y_start = (0, 0) if show_walls else (1, 1) + + if show_walls: + x_end, y_end = self.width, self.height + else: + x_end, y_end = self.width - 1, self.height - 1 + + for x in range(x_start, x_end): + row = [] + for y in range(y_start, y_end): + row.append(self.list_things_at((x, y))) + result.append(row) + return result + + def percepts_from(self, agent, location, tclass=Thing): + """Return percepts from a given location, + and replaces some items with percepts from chapter 7.""" + thing_percepts = { + Gold: Glitter(), + Wall: Bump(), + Wumpus: Stench(), + Pit: Breeze()} + + """Agents don't need to get their percepts""" + thing_percepts[agent.__class__] = None + + """Gold only glitters in its cell""" + if location != agent.location: + thing_percepts[Gold] = None + + result = [thing_percepts.get(thing.__class__, thing) for thing in self.things + if thing.location == location and isinstance(thing, tclass)] + return result if len(result) else [None] + + def percept(self, agent): + """Return things in adjacent (not diagonal) cells of the agent. + Result format: [Left, Right, Up, Down, Center / Current location]""" + x, y = agent.location + result = [] + result.append(self.percepts_from(agent, (x - 1, y))) + result.append(self.percepts_from(agent, (x + 1, y))) + result.append(self.percepts_from(agent, (x, y - 1))) + result.append(self.percepts_from(agent, (x, y + 1))) + result.append(self.percepts_from(agent, (x, y))) + + """The wumpus gives out a loud scream once it's killed.""" + wumpus = [thing for thing in self.things if isinstance(thing, Wumpus)] + if len(wumpus) and not wumpus[0].alive and not wumpus[0].screamed: + result[-1].append(Scream()) + wumpus[0].screamed = True + + return result + + def execute_action(self, agent, action): + """Modify the state of the environment based on the agent's actions. + Performance score taken directly out of the book.""" + + if isinstance(agent, Explorer) and self.in_danger(agent): + return + + agent.bump = False + if action in ['TurnRight', 'TurnLeft', 'Forward', 'Grab']: + super().execute_action(agent, action) + agent.performance -= 1 + elif action == 'Climb': + if agent.location == (1, 1): # Agent can only climb out of (1,1) + agent.performance += 1000 if Gold() in agent.holding else 0 + self.delete_thing(agent) + elif action == 'Shoot': + """The arrow travels straight down the path the agent is facing""" + if agent.has_arrow: + arrow_travel = agent.direction.move_forward(agent.location) + while self.is_inbounds(arrow_travel): + wumpus = [thing for thing in self.list_things_at(arrow_travel) + if isinstance(thing, Wumpus)] + if len(wumpus): + wumpus[0].alive = False + break + arrow_travel = agent.direction.move_forward(agent.location) + agent.has_arrow = False + + def in_danger(self, agent): + """Check if Explorer is in danger (Pit or Wumpus), if he is, kill him""" + for thing in self.list_things_at(agent.location): + if isinstance(thing, Pit) or (isinstance(thing, Wumpus) and thing.alive): + agent.alive = False + agent.performance -= 1000 + agent.killed_by = thing.__class__.__name__ + return True + return False + + def is_done(self): + """The game is over when the Explorer is killed + or if he climbs out of the cave only at (1,1).""" + explorer = [agent for agent in self.agents if isinstance(agent, Explorer)] + if len(explorer): + if explorer[0].alive: + return False + else: + print("Death by {} [-1000].".format(explorer[0].killed_by)) + else: + print("Explorer climbed out {}." + .format("with Gold [+1000]!" if Gold() not in self.things else "without Gold [+0]")) + return True + + # TODO: Arrow needs to be implemented + + +# ______________________________________________________________________________ + + +def compare_agents(EnvFactory, AgentFactories, n=10, steps=1000): + """See how well each of several agents do in n instances of an environment. + Pass in a factory (constructor) for environments, and several for agents. + Create n instances of the environment, and run each agent in copies of + each one for steps. Return a list of (agent, average-score) tuples. + >>> environment = TrivialVacuumEnvironment + >>> agents = [ModelBasedVacuumAgent, ReflexVacuumAgent] + >>> result = compare_agents(environment, agents) + >>> performance_ModelBasedVacuumAgent = result[0][1] + >>> performance_ReflexVacuumAgent = result[1][1] + >>> performance_ReflexVacuumAgent <= performance_ModelBasedVacuumAgent + True + """ + envs = [EnvFactory() for i in range(n)] + return [(A, test_agent(A, steps, copy.deepcopy(envs))) + for A in AgentFactories] + + +def test_agent(AgentFactory, steps, envs): + """Return the mean score of running an agent in each of the envs, for steps + >>> def constant_prog(percept): + ... return percept + ... + >>> agent = Agent(constant_prog) + >>> result = agent.program(5) + >>> result == 5 + True + """ + + def score(env): + agent = AgentFactory() + env.add_thing(agent) + env.run(steps) + return agent.performance + + return mean(map(score, envs)) + + +# _________________________________________________________________________ + + +__doc__ += """ +>>> a = ReflexVacuumAgent() +>>> a.program((loc_A, 'Clean')) +'Right' +>>> a.program((loc_B, 'Clean')) +'Left' +>>> a.program((loc_A, 'Dirty')) +'Suck' +>>> a.program((loc_A, 'Dirty')) +'Suck' + +>>> e = TrivialVacuumEnvironment() +>>> e.add_thing(ModelBasedVacuumAgent()) +>>> e.run(5) + +""" diff --git a/q-learning-9b/aima/csp.py b/q-learning-9b/aima/csp.py new file mode 100644 index 0000000..a4caec5 --- /dev/null +++ b/q-learning-9b/aima/csp.py @@ -0,0 +1,1502 @@ +"""CSP (Constraint Satisfaction Problems) problems and solvers. (Chapter 6)""" + +import itertools +import random +import re +import string +from collections import defaultdict, Counter +from functools import reduce +from operator import eq, neg + +from sortedcontainers import SortedSet + +from aima import search +from aima.utils import argmin_random_tie, count, first, extend, flatten + + +class CSP(search.Problem): + """This class describes finite-domain Constraint Satisfaction Problems. + A CSP is specified by the following inputs:: + + variables A list of variables; each is atomic (e.g. int or string). + domains A dict of {var:[possible_value, ...]} entries. + neighbors A dict of {var:[var,...]} that for each variable lists + the other variables that participate in constraints. + constraints A function f(A, a, B, b) that returns true if neighbors + A, B satisfy the constraint when they have values A=a, B=b + + In the textbook and in most mathematical definitions, the + constraints are specified as explicit pairs of allowable values, + but the formulation here is easier to express and more compact for + most cases (for example, the n-Queens problem can be represented + in O(n) space using this notation, instead of O(n^4) for the + explicit representation). In terms of describing the CSP as a + problem, that's all there is. + + However, the class also supports data structures and methods that help you + solve CSPs by calling a search function on the CSP. Methods and slots are + as follows, where the argument 'a' represents an assignment, which is a + dict of {var:val} entries:: + + assign(var, val, a) Assign a[var] = val; do other bookkeeping + unassign(var, a) Do del a[var], plus other bookkeeping + nconflicts(var, val, a) Return the number of other variables that + conflict with var=val + curr_domains[var] Slot: remaining consistent values for var + Used by constraint propagation routines. + + The following methods are used only by graph_search and tree_search:: + + actions(state) Return a list of actions + result(state, action) Return a successor of state + goal_test(state) Return true if all constraints satisfied + + The following are just for debugging purposes:: + + nassigns Slot: tracks the number of assignments made + display(a) Print a human-readable representation + """ + + def __init__(self, variables, domains, neighbors, constraints): + """Construct a CSP problem. If variables is empty, it becomes domains.keys().""" + super().__init__(()) + variables = variables or list(domains.keys()) + self.variables = variables + self.domains = domains + self.neighbors = neighbors + # store the constraint relation privately and expose it through the + # constraints() method below, which counts every consistency check + self.constraint_relation = constraints + self.curr_domains = None + self.nassigns = 0 + self.nchecks = 0 + + def constraints(self, A, a, B, b): + """Return True if neighbors A, B satisfy the constraint when A=a, B=b. + Every call is tallied in self.nchecks, so callers can measure the number + of consistency checks performed (e.g. to reproduce the benchmarks in + Figure 6.1), just as self.nassigns tallies the assignments made.""" + self.nchecks += 1 + return self.constraint_relation(A, a, B, b) + + def assign(self, var, val, assignment): + """Add {var: val} to assignment; Discard the old value if any.""" + assignment[var] = val + self.nassigns += 1 + + def unassign(self, var, assignment): + """Remove {var: val} from assignment. + DO NOT call this if you are changing a variable to a new value; + just call assign for that.""" + if var in assignment: + del assignment[var] + + def nconflicts(self, var, val, assignment): + """Return the number of conflicts var=val has with other variables.""" + + # Subclasses may implement this more efficiently + def conflict(var2): + return var2 in assignment and not self.constraints(var, val, var2, assignment[var2]) + + return count(conflict(v) for v in self.neighbors[var]) + + def count_lost_values(self, var, val, assignment): + """Return how many values would be ruled out in the domains of the + unassigned neighbours of var if var were assigned val (the count used + by the least-constraining-value heuristic).""" + return count(not self.constraints(var, val, neighbor, dval) + for neighbor in self.neighbors[var] if neighbor not in assignment + for dval in self.domains[neighbor]) + + def display(self, assignment): + """Show a human-readable representation of the CSP.""" + # Subclasses can print in a prettier way, or display with a GUI + print(assignment) + + # These methods are for the tree and graph-search interface: + + def actions(self, state): + """Return a list of applicable actions: non conflicting + assignments to an unassigned variable.""" + if len(state) == len(self.variables): + return [] + else: + assignment = dict(state) + var = first([v for v in self.variables if v not in assignment]) + return [(var, val) for val in self.domains[var] + if self.nconflicts(var, val, assignment) == 0] + + def result(self, state, action): + """Perform an action and return the new state.""" + (var, val) = action + return state + ((var, val),) + + def goal_test(self, state): + """The goal is to assign all variables, with all constraints satisfied.""" + assignment = dict(state) + return (len(assignment) == len(self.variables) + and all(self.nconflicts(variables, assignment[variables], assignment) == 0 + for variables in self.variables)) + + # These are for constraint propagation + + def support_pruning(self): + """Make sure we can prune values from domains. (We want to pay + for this only if we use it.)""" + if self.curr_domains is None: + self.curr_domains = {v: list(self.domains[v]) for v in self.variables} + + def suppose(self, var, value): + """Start accumulating inferences from assuming var=value.""" + self.support_pruning() + removals = [(var, a) for a in self.curr_domains[var] if a != value] + self.curr_domains[var] = [value] + return removals + + def prune(self, var, value, removals): + """Rule out var=value.""" + self.curr_domains[var].remove(value) + if removals is not None: + removals.append((var, value)) + + def choices(self, var): + """Return all values for var that aren't currently ruled out.""" + return (self.curr_domains or self.domains)[var] + + def infer_assignment(self): + """Return the partial assignment implied by the current inferences.""" + self.support_pruning() + return {v: self.curr_domains[v][0] + for v in self.variables if 1 == len(self.curr_domains[v])} + + def restore(self, removals): + """Undo a supposition and all inferences from it.""" + for B, b in removals: + self.curr_domains[B].append(b) + + # This is for min_conflicts search + + def conflicted_vars(self, current): + """Return a list of variables in current assignment that are in conflict""" + return [var for var in self.variables + if self.nconflicts(var, current[var], current) > 0] + + +# ______________________________________________________________________________ +# Constraint Propagation with AC3 + + +def no_arc_heuristic(csp, queue): + """Return the arc queue unchanged (no ordering heuristic for AC3).""" + return queue + + +def dom_j_up(csp, queue): + """Order the arc queue so arcs whose second variable has the smallest current + domain are popped first (a SortedSet keyed by the negated domain size).""" + return SortedSet(queue, key=lambda t: neg(len(csp.curr_domains[t[1]]))) + + +def AC3(csp, queue=None, removals=None, arc_heuristic=dom_j_up): + """[Figure 6.3]""" + if queue is None: + queue = {(Xi, Xk) for Xi in csp.variables for Xk in csp.neighbors[Xi]} + csp.support_pruning() + queue = arc_heuristic(csp, queue) + checks = 0 + while queue: + (Xi, Xj) = queue.pop() + revised, checks = revise(csp, Xi, Xj, removals, checks) + if revised: + if not csp.curr_domains[Xi]: + return False, checks # CSP is inconsistent + for Xk in csp.neighbors[Xi]: + if Xk != Xj: + queue.add((Xk, Xi)) + return True, checks # CSP is arc-consistent + + +def revise(csp, Xi, Xj, removals, checks=0): + """Return true if we remove a value.""" + revised = False + for x in csp.curr_domains[Xi][:]: + # If Xi=x conflicts with Xj=y for every possible y, eliminate Xi=x + # if all(not csp.constraints(Xi, x, Xj, y) for y in csp.curr_domains[Xj]): + conflict = True + for y in csp.curr_domains[Xj]: + if csp.constraints(Xi, x, Xj, y): + conflict = False + checks += 1 + if not conflict: + break + if conflict: + csp.prune(Xi, x, removals) + revised = True + return revised, checks + + +# Constraint Propagation with AC3b: an improved version +# of AC3 with double-support domain-heuristic + +def AC3b(csp, queue=None, removals=None, arc_heuristic=dom_j_up): + """An improved version of AC3 that uses double-support checks to share work + between an arc (Xi, Xj) and its reverse (Xj, Xi). Returns a tuple + (is_consistent, checks): False if a domain is wiped out, True if the CSP is + made arc-consistent, where checks counts the consistency checks performed.""" + if queue is None: + queue = {(Xi, Xk) for Xi in csp.variables for Xk in csp.neighbors[Xi]} + csp.support_pruning() + queue = arc_heuristic(csp, queue) + checks = 0 + while queue: + (Xi, Xj) = queue.pop() + # Si_p values are all known to be supported by Xj + # Sj_p values are all known to be supported by Xi + # Dj - Sj_p = Sj_u values are unknown, as yet, to be supported by Xi + Si_p, Sj_p, Sj_u, checks = partition(csp, Xi, Xj, checks) + if not Si_p: + return False, checks # CSP is inconsistent + revised = False + for x in set(csp.curr_domains[Xi]) - Si_p: + csp.prune(Xi, x, removals) + revised = True + if revised: + for Xk in csp.neighbors[Xi]: + if Xk != Xj: + queue.add((Xk, Xi)) + if (Xj, Xi) in queue: + if isinstance(queue, set): + # or queue -= {(Xj, Xi)} or queue.remove((Xj, Xi)) + queue.difference_update({(Xj, Xi)}) + else: + queue.difference_update((Xj, Xi)) + # the elements in D_j which are supported by Xi are given by the union of Sj_p with the set of those + # elements of Sj_u which further processing will show to be supported by some vi_p in Si_p + for vj_p in Sj_u: + for vi_p in Si_p: + conflict = True + if csp.constraints(Xj, vj_p, Xi, vi_p): + conflict = False + Sj_p.add(vj_p) + checks += 1 + if not conflict: + break + revised = False + for x in set(csp.curr_domains[Xj]) - Sj_p: + csp.prune(Xj, x, removals) + revised = True + if revised: + for Xk in csp.neighbors[Xj]: + if Xk != Xi: + queue.add((Xk, Xj)) + return True, checks # CSP is arc-consistent + + +def partition(csp, Xi, Xj, checks=0): + """Helper for AC3b: split the domains of Xi and Xj using double-support checks. + Returns (Si_p, Sj_p, Sj_u, checks) where Si_p is the set of Xi values supported + by Xj, Sj_p the set of Xj values already known to be supported by Xi, Sj_u the + remaining (as yet unconfirmed) Xj values, and checks the updated check count.""" + Si_p = set() + Sj_p = set() + Sj_u = set(csp.curr_domains[Xj]) + for vi_u in csp.curr_domains[Xi]: + conflict = True + # now, in order to establish support for a value vi_u in Di it seems better to try to find a support among + # the values in Sj_u first, because for each vj_u in Sj_u the check (vi_u, vj_u) is a double-support check + # and it is just as likely that any vj_u in Sj_u supports vi_u than it is that any vj_p in Sj_p does... + for vj_u in Sj_u - Sj_p: + # double-support check + if csp.constraints(Xi, vi_u, Xj, vj_u): + conflict = False + Si_p.add(vi_u) + Sj_p.add(vj_u) + checks += 1 + if not conflict: + break + # ... and only if no support can be found among the elements in Sj_u, should the elements vj_p in Sj_p be used + # for single-support checks (vi_u, vj_p) + if conflict: + for vj_p in Sj_p: + # single-support check + if csp.constraints(Xi, vi_u, Xj, vj_p): + conflict = False + Si_p.add(vi_u) + checks += 1 + if not conflict: + break + return Si_p, Sj_p, Sj_u - Sj_p, checks + + +# Constraint Propagation with AC4 + +def AC4(csp, queue=None, removals=None, arc_heuristic=dom_j_up): + """Enforce arc consistency using AC4, which keeps per-value support counters so + that pruning a value only triggers re-checks of the values it supported. Returns + a tuple (is_consistent, checks): False if a domain is wiped out, True if the CSP + is made arc-consistent, where checks counts the consistency checks performed.""" + if queue is None: + queue = {(Xi, Xk) for Xi in csp.variables for Xk in csp.neighbors[Xi]} + csp.support_pruning() + queue = arc_heuristic(csp, queue) + support_counter = Counter() + variable_value_pairs_supported = defaultdict(set) + unsupported_variable_value_pairs = [] + checks = 0 + # construction and initialization of support sets + while queue: + (Xi, Xj) = queue.pop() + revised = False + for x in csp.curr_domains[Xi][:]: + for y in csp.curr_domains[Xj]: + if csp.constraints(Xi, x, Xj, y): + support_counter[(Xi, x, Xj)] += 1 + variable_value_pairs_supported[(Xj, y)].add((Xi, x)) + checks += 1 + if support_counter[(Xi, x, Xj)] == 0: + csp.prune(Xi, x, removals) + revised = True + unsupported_variable_value_pairs.append((Xi, x)) + if revised: + if not csp.curr_domains[Xi]: + return False, checks # CSP is inconsistent + # propagation of removed values + while unsupported_variable_value_pairs: + Xj, y = unsupported_variable_value_pairs.pop() + for Xi, x in variable_value_pairs_supported[(Xj, y)]: + revised = False + if x in csp.curr_domains[Xi][:]: + support_counter[(Xi, x, Xj)] -= 1 + if support_counter[(Xi, x, Xj)] == 0: + csp.prune(Xi, x, removals) + revised = True + unsupported_variable_value_pairs.append((Xi, x)) + if revised: + if not csp.curr_domains[Xi]: + return False, checks # CSP is inconsistent + return True, checks # CSP is arc-consistent + + +# ______________________________________________________________________________ +# CSP Backtracking Search + +# Variable ordering + + +def first_unassigned_variable(assignment, csp): + """The default variable order.""" + return first([var for var in csp.variables if var not in assignment]) + + +def mrv(assignment, csp): + """Minimum-remaining-values heuristic.""" + return argmin_random_tie([v for v in csp.variables if v not in assignment], + key=lambda var: num_legal_values(csp, var, assignment)) + + +def num_legal_values(csp, var, assignment): + """Return how many values are still legal for var (used by the mrv heuristic): + the size of its current domain if pruning is active, otherwise the number of + domain values that conflict with no other variable in the assignment.""" + if csp.curr_domains: + return len(csp.curr_domains[var]) + else: + return count(csp.nconflicts(var, val, assignment) == 0 for val in csp.domains[var]) + + +# Value ordering + + +def unordered_domain_values(var, assignment, csp): + """The default value order.""" + return csp.choices(var) + + +def lcv(var, assignment, csp): + """Least-constraining-values heuristic.""" + return sorted(csp.choices(var), key=lambda val: csp.count_lost_values(var, val, assignment)) + + +# Inference + + +def no_inference(csp, var, value, assignment, removals): + """The default inference step for backtracking: do nothing and always succeed.""" + return True + + +def forward_checking(csp, var, value, assignment, removals): + """Prune neighbor values inconsistent with var=value.""" + csp.support_pruning() + for B in csp.neighbors[var]: + if B not in assignment: + for b in csp.curr_domains[B][:]: + if not csp.constraints(var, value, B, b): + csp.prune(B, b, removals) + if not csp.curr_domains[B]: + return False + return True + + +def mac(csp, var, value, assignment, removals, constraint_propagation=AC3b): + """Maintain arc consistency.""" + return constraint_propagation(csp, {(X, var) for X in csp.neighbors[var]}, removals) + + +# The search, proper + + +def backtracking_search(csp, select_unassigned_variable=first_unassigned_variable, + order_domain_values=unordered_domain_values, inference=no_inference): + """[Figure 6.5]""" + + def backtrack(assignment): + if len(assignment) == len(csp.variables): + return assignment + var = select_unassigned_variable(assignment, csp) + for value in order_domain_values(var, assignment, csp): + if 0 == csp.nconflicts(var, value, assignment): + csp.assign(var, value, assignment) + removals = csp.suppose(var, value) + if inference(csp, var, value, assignment, removals): + result = backtrack(assignment) + if result is not None: + return result + csp.restore(removals) + csp.unassign(var, assignment) + return None + + result = backtrack({}) + assert result is None or csp.goal_test(result) + return result + + +# ______________________________________________________________________________ +# Min-conflicts Hill Climbing search for CSPs + + +def min_conflicts(csp, max_steps=100000): + """Solve a CSP by stochastic Hill Climbing on the number of conflicts.""" + # Generate a complete assignment for all variables (probably with conflicts) + csp.current = current = {} + for var in csp.variables: + val = min_conflicts_value(csp, var, current) + csp.assign(var, val, current) + # Now repeatedly choose a random conflicted variable and change it + for i in range(max_steps): + conflicted = csp.conflicted_vars(current) + if not conflicted: + return current + var = random.choice(conflicted) + val = min_conflicts_value(csp, var, current) + csp.assign(var, val, current) + return None + + +def min_conflicts_value(csp, var, current): + """Return the value that will give var the least number of conflicts. + If there is a tie, choose at random.""" + return argmin_random_tie(csp.domains[var], key=lambda val: csp.nconflicts(var, val, current)) + + +# ______________________________________________________________________________ + + +def tree_csp_solver(csp): + """[Figure 6.11] Solve a tree-structured CSP in linear time: order the variables + so each has at most one parent, make the tree directionally arc-consistent from + the leaves up, then assign each variable consistently from the root down.""" + assignment = {} + root = csp.variables[0] + X, parent = topological_sort(csp, root) + + csp.support_pruning() + for Xj in reversed(X[1:]): + if not make_arc_consistent(parent[Xj], Xj, csp): + return None + + assignment[root] = csp.curr_domains[root][0] + for Xi in X[1:]: + assignment[Xi] = assign_value(parent[Xi], Xi, csp, assignment) + if not assignment[Xi]: + return None + return assignment + + +def topological_sort(X, root): + """Returns the topological sort of X starting from the root. + + Input: + X is a list with the nodes of the graph + N is the dictionary with the neighbors of each node + root denotes the root of the graph. + + Output: + stack is a list with the nodes topologically sorted + parents is a dictionary pointing to each node's parent + + Other: + visited shows the state (visited - not visited) of nodes + + """ + neighbors = X.neighbors + + visited = defaultdict(lambda: False) + + stack = [] + parents = {} + + build_topological(root, None, neighbors, visited, stack, parents) + return stack, parents + + +def build_topological(node, parent, neighbors, visited, stack, parents): + """Build the topological sort and the parents of each node in the graph.""" + visited[node] = True + + for n in neighbors[node]: + if not visited[n]: + build_topological(n, node, neighbors, visited, stack, parents) + + parents[node] = parent + stack.insert(0, node) + + +def make_arc_consistent(Xj, Xk, csp): + """Make arc between parent (Xj) and child (Xk) consistent under the csp's constraints, + by removing the possible values of Xj that cause inconsistencies.""" + # csp.curr_domains[Xj] = [] + for val1 in csp.domains[Xj]: + keep = False # Keep or remove val1 + for val2 in csp.domains[Xk]: + if csp.constraints(Xj, val1, Xk, val2): + # Found a consistent assignment for val1, keep it + keep = True + break + + if not keep: + # Remove val1 + csp.prune(Xj, val1, None) + + return csp.curr_domains[Xj] + + +def assign_value(Xj, Xk, csp, assignment): + """Assign a value to Xk given Xj's (Xk's parent) assignment. + Return the first value that satisfies the constraints.""" + parent_assignment = assignment[Xj] + for val in csp.curr_domains[Xk]: + if csp.constraints(Xj, parent_assignment, Xk, val): + return val + + # No consistent assignment available + return None + + +# ______________________________________________________________________________ +# Map Coloring CSP Problems + + +class UniversalDict: + """A universal dict maps any key to the same value. We use it here + as the domains dict for CSPs in which all variables have the same domain. + >>> d = UniversalDict(42) + >>> d['life'] + 42 + """ + + def __init__(self, value): self.value = value + + def __getitem__(self, key): return self.value + + def __repr__(self): return '{{Any: {0!r}}}'.format(self.value) + + +def different_values_constraint(A, a, B, b): + """A constraint saying two neighboring variables must differ in value.""" + return a != b + + +def MapColoringCSP(colors, neighbors): + """Make a CSP for the problem of coloring a map with different colors + for any two adjacent regions. Arguments are a list of colors, and a + dict of {region: [neighbor,...]} entries. This dict may also be + specified as a string of the form defined by parse_neighbors.""" + if isinstance(neighbors, str): + neighbors = parse_neighbors(neighbors) + return CSP(list(neighbors.keys()), UniversalDict(colors), neighbors, different_values_constraint) + + +def parse_neighbors(neighbors): + """Convert a string of the form 'X: Y Z; Y: Z' into a dict mapping + regions to neighbors. The syntax is a region name followed by a ':' + followed by zero or more region names, followed by ';', repeated for + each region name. If you say 'X: Y' you don't need 'Y: X'. + >>> parse_neighbors('X: Y Z; Y: Z') == {'Y': ['X', 'Z'], 'X': ['Y', 'Z'], 'Z': ['X', 'Y']} + True + """ + dic = defaultdict(list) + specs = [spec.split(':') for spec in neighbors.split(';')] + for (A, Aneighbors) in specs: + A = A.strip() + for B in Aneighbors.split(): + dic[A].append(B) + dic[B].append(A) + return dic + + +australia_csp = MapColoringCSP(list('RGB'), """SA: WA NT Q NSW V; NT: WA Q; NSW: Q V; T: """) + +usa_csp = MapColoringCSP(list('RGBY'), + """WA: OR ID; OR: ID NV CA; CA: NV AZ; NV: ID UT AZ; ID: MT WY UT; + UT: WY CO AZ; MT: ND SD WY; WY: SD NE CO; CO: NE KA OK NM; NM: OK TX AZ; + ND: MN SD; SD: MN IA NE; NE: IA MO KA; KA: MO OK; OK: MO AR TX; + TX: AR LA; MN: WI IA; IA: WI IL MO; MO: IL KY TN AR; AR: MS TN LA; + LA: MS; WI: MI IL; IL: IN KY; IN: OH KY; MS: TN AL; AL: TN GA FL; + MI: OH IN; OH: PA WV KY; KY: WV VA TN; TN: VA NC GA; GA: NC SC FL; + PA: NY NJ DE MD WV; WV: MD VA; VA: MD DC NC; NC: SC; NY: VT MA CT NJ; + NJ: DE; DE: MD; MD: DC; VT: NH MA; MA: NH RI CT; CT: RI; ME: NH; + HI: ; AK: """) + +france_csp = MapColoringCSP(list('RGBY'), + """AL: LO FC; AQ: MP LI PC; AU: LI CE BO RA LR MP; BO: CE IF CA FC RA + AU; BR: NB PL; CA: IF PI LO FC BO; CE: PL NB NH IF BO AU LI PC; FC: BO + CA LO AL RA; IF: NH PI CA BO CE; LI: PC CE AU MP AQ; LO: CA AL FC; LR: + MP AU RA PA; MP: AQ LI AU LR; NB: NH CE PL BR; NH: PI IF CE NB; NO: + PI; PA: LR RA; PC: PL CE LI AQ; PI: NH NO CA IF; PL: BR NB CE PC; RA: + AU BO FC PA LR""") + + +# ______________________________________________________________________________ +# n-Queens Problem + + +def queen_constraint(A, a, B, b): + """Constraint is satisfied (true) if A, B are really the same variable, + or if they are not in the same row, down diagonal, or up diagonal.""" + return A == B or (a != b and A + a != B + b and A - a != B - b) + + +class NQueensCSP(CSP): + r""" + Make a CSP for the nQueens problem for search with min_conflicts. + Suitable for large n, it uses only data structures of size O(n). + Think of placing queens one per column, from left to right. + That means position (x, y) represents (var, val) in the CSP. + The main structures are three arrays to count queens that could conflict:: + + rows[i] Number of queens in the ith row (i.e. val == i) + downs[i] Number of queens in the \ diagonal + such that their (x, y) coordinates sum to i + ups[i] Number of queens in the / diagonal + such that their (x, y) coordinates have x-y+n-1 = i + + We increment/decrement these counts each time a queen is placed/moved from + a row/diagonal. So moving is O(1), as is nconflicts. But choosing + a variable, and a best value for the variable, are each O(n). + If you want, you can keep track of conflicted variables, then variable + selection will also be O(1). + + >>> len(backtracking_search(NQueensCSP(8))) + 8 + """ + + def __init__(self, n): + """Initialize data structures for n Queens.""" + CSP.__init__(self, list(range(n)), UniversalDict(list(range(n))), + UniversalDict(list(range(n))), queen_constraint) + + self.rows = [0] * n + self.ups = [0] * (2 * n - 1) + self.downs = [0] * (2 * n - 1) + + def nconflicts(self, var, val, assignment): + """The number of conflicts, as recorded with each assignment. + Count conflicts in row and in up, down diagonals. If there + is a queen there, it can't conflict with itself, so subtract 3.""" + n = len(self.variables) + c = self.rows[val] + self.downs[var + val] + self.ups[var - val + n - 1] + if assignment.get(var, None) == val: + c -= 3 + return c + + def assign(self, var, val, assignment): + """Assign var, and keep track of conflicts.""" + old_val = assignment.get(var, None) + if val != old_val: + if old_val is not None: # Remove old val if there was one + self.record_conflict(assignment, var, old_val, -1) + self.record_conflict(assignment, var, val, +1) + CSP.assign(self, var, val, assignment) + + def unassign(self, var, assignment): + """Remove var from assignment (if it is there) and track conflicts.""" + if var in assignment: + self.record_conflict(assignment, var, assignment[var], -1) + CSP.unassign(self, var, assignment) + + def record_conflict(self, assignment, var, val, delta): + """Record conflicts caused by addition or deletion of a Queen.""" + n = len(self.variables) + self.rows[val] += delta + self.downs[var + val] += delta + self.ups[var - val + n - 1] += delta + + def display(self, assignment): + """Print the queens and the nconflicts values (for debugging).""" + n = len(self.variables) + for val in range(n): + for var in range(n): + if assignment.get(var, '') == val: + ch = 'Q' + elif (var + val) % 2 == 0: + ch = '.' + else: + ch = '-' + print(ch, end=' ') + print(' ', end=' ') + for var in range(n): + if assignment.get(var, '') == val: + ch = '*' + else: + ch = ' ' + print(str(self.nconflicts(var, val, assignment)) + ch, end=' ') + print() + + +# ______________________________________________________________________________ +# Sudoku + + +easy1 = '..3.2.6..9..3.5..1..18.64....81.29..7.......8..67.82....26.95..8..2.3..9..5.1.3..' +harder1 = '4173698.5.3..........7......2.....6.....8.4......1.......6.3.7.5..2.....1.4......' + +_R3 = list(range(3)) +_CELL = itertools.count().__next__ +_BGRID = [[[[_CELL() for x in _R3] for y in _R3] for bx in _R3] for by in _R3] +_BOXES = flatten([list(map(flatten, brow)) for brow in _BGRID]) +_ROWS = flatten([list(map(flatten, zip(*brow))) for brow in _BGRID]) +_COLS = list(zip(*_ROWS)) + +_NEIGHBORS = {v: set() for v in flatten(_ROWS)} +for unit in map(set, _BOXES + _ROWS + _COLS): + for v in unit: + _NEIGHBORS[v].update(unit - {v}) + + +class Sudoku(CSP): + """ + A Sudoku problem. + The box grid is a 3x3 array of boxes, each a 3x3 array of cells. + Each cell holds a digit in 1..9. In each box, all digits are + different; the same for each row and column as a 9x9 grid. + >>> e = Sudoku(easy1) + >>> e.display(e.infer_assignment()) + . . 3 | . 2 . | 6 . . + 9 . . | 3 . 5 | . . 1 + . . 1 | 8 . 6 | 4 . . + ------+-------+------ + . . 8 | 1 . 2 | 9 . . + 7 . . | . . . | . . 8 + . . 6 | 7 . 8 | 2 . . + ------+-------+------ + . . 2 | 6 . 9 | 5 . . + 8 . . | 2 . 3 | . . 9 + . . 5 | . 1 . | 3 . . + >>> AC3(e) # doctest: +ELLIPSIS + (True, ...) + >>> e.display(e.infer_assignment()) + 4 8 3 | 9 2 1 | 6 5 7 + 9 6 7 | 3 4 5 | 8 2 1 + 2 5 1 | 8 7 6 | 4 9 3 + ------+-------+------ + 5 4 8 | 1 3 2 | 9 7 6 + 7 2 9 | 5 6 4 | 1 3 8 + 1 3 6 | 7 9 8 | 2 4 5 + ------+-------+------ + 3 7 2 | 6 8 9 | 5 1 4 + 8 1 4 | 2 5 3 | 7 6 9 + 6 9 5 | 4 1 7 | 3 8 2 + >>> h = Sudoku(harder1) + >>> backtracking_search(h, select_unassigned_variable=mrv, inference=forward_checking) is not None + True + """ + + R3 = _R3 + Cell = _CELL + bgrid = _BGRID + boxes = _BOXES + rows = _ROWS + cols = _COLS + neighbors = _NEIGHBORS + + def __init__(self, grid): + """Build a Sudoku problem from a string representing the grid: + the digits 1-9 denote a filled cell, '.' or '0' an empty one; + other characters are ignored.""" + squares = iter(re.findall(r'\d|\.', grid)) + domains = {var: [ch] if ch in '123456789' else '123456789' + for var, ch in zip(flatten(self.rows), squares)} + for _ in squares: + raise ValueError("Not a Sudoku grid", grid) # Too many squares + CSP.__init__(self, None, domains, self.neighbors, different_values_constraint) + + def display(self, assignment): + """Print the Sudoku grid for the given assignment as a 9x9 board.""" + + def show_box(box): return [' '.join(map(show_cell, row)) for row in box] + + def show_cell(cell): return str(assignment.get(cell, '.')) + + def abut(lines1, lines2): return list( + map(' | '.join, list(zip(lines1, lines2)))) + + print('\n------+-------+------\n'.join( + '\n'.join(reduce( + abut, map(show_box, brow))) for brow in self.bgrid)) + + +# ______________________________________________________________________________ +# The Zebra Puzzle + + +def Zebra(): + """Return an instance of the Zebra Puzzle.""" + Colors = 'Red Yellow Blue Green Ivory'.split() + Pets = 'Dog Fox Snails Horse Zebra'.split() + Drinks = 'OJ Tea Coffee Milk Water'.split() + Countries = 'Englishman Spaniard Norwegian Ukrainian Japanese'.split() + Smokes = 'Kools Chesterfields Winston LuckyStrike Parliaments'.split() + variables = Colors + Pets + Drinks + Countries + Smokes + domains = {} + for var in variables: + domains[var] = list(range(1, 6)) + domains['Norwegian'] = [1] + domains['Milk'] = [3] + neighbors = parse_neighbors("""Englishman: Red; + Spaniard: Dog; Kools: Yellow; Chesterfields: Fox; + Norwegian: Blue; Winston: Snails; LuckyStrike: OJ; + Ukrainian: Tea; Japanese: Parliaments; Kools: Horse; + Coffee: Green; Green: Ivory""") + for type in [Colors, Pets, Drinks, Countries, Smokes]: + for A in type: + for B in type: + if A != B: + if B not in neighbors[A]: + neighbors[A].append(B) + if A not in neighbors[B]: + neighbors[B].append(A) + + def zebra_constraint(A, a, B, b, recurse=0): + same = (a == b) + next_to = abs(a - b) == 1 + if A == 'Englishman' and B == 'Red': + return same + if A == 'Spaniard' and B == 'Dog': + return same + if A == 'Chesterfields' and B == 'Fox': + return next_to + if A == 'Norwegian' and B == 'Blue': + return next_to + if A == 'Kools' and B == 'Yellow': + return same + if A == 'Winston' and B == 'Snails': + return same + if A == 'LuckyStrike' and B == 'OJ': + return same + if A == 'Ukrainian' and B == 'Tea': + return same + if A == 'Japanese' and B == 'Parliaments': + return same + if A == 'Kools' and B == 'Horse': + return next_to + if A == 'Coffee' and B == 'Green': + return same + if A == 'Green' and B == 'Ivory': + return a - 1 == b + if recurse == 0: + return zebra_constraint(B, b, A, a, 1) + if ((A in Colors and B in Colors) or + (A in Pets and B in Pets) or + (A in Drinks and B in Drinks) or + (A in Countries and B in Countries) or + (A in Smokes and B in Smokes)): + return not same + raise Exception('error') + + return CSP(variables, domains, neighbors, zebra_constraint) + + +def solve_zebra(algorithm=min_conflicts, **args): + """Solve the Zebra Puzzle with the given CSP algorithm, print which person is in + each house, and return a tuple (zebra_owner_house, water_drinker_house, nassigns, + full_assignment).""" + z = Zebra() + ans = algorithm(z, **args) + for h in range(1, 6): + print('House', h, end=' ') + for (var, val) in ans.items(): + if val == h: + print(var, end=' ') + print() + return ans['Zebra'], ans['Water'], z.nassigns, ans + + +# ______________________________________________________________________________ +# n-ary Constraint Satisfaction Problem + +class NaryCSP: + """ + A nary-CSP consists of: + domains : a dictionary that maps each variable to its domain + constraints : a list of constraints + variables : a set of variables + var_to_const: a variable to set of constraints dictionary + """ + + def __init__(self, domains, constraints): + """Domains is a variable:domain dictionary + constraints is a list of constraints + """ + self.variables = set(domains) + self.domains = domains + self.constraints = constraints + self.var_to_const = {var: set() for var in self.variables} + for con in constraints: + for var in con.scope: + self.var_to_const[var].add(con) + + def __str__(self): + """String representation of CSP""" + return str(self.domains) + + def display(self, assignment=None): + """More detailed string representation of CSP""" + if assignment is None: + assignment = {} + print(assignment) + + def consistent(self, assignment): + """assignment is a variable:value dictionary + + returns True if all of the constraints that can be evaluated + evaluate to True given assignment. + """ + return all(con.holds(assignment) + for con in self.constraints + if all(v in assignment for v in con.scope)) + + +class Constraint: + """ + A Constraint consists of: + scope : a tuple of variables + condition: a function that can applied to a tuple of values + for the variables. + """ + + def __init__(self, scope, condition): + self.scope = scope + self.condition = condition + + def __repr__(self): + return self.condition.__name__ + str(self.scope) + + def holds(self, assignment): + """Returns the value of Constraint con evaluated in assignment. + + precondition: all variables are assigned in assignment + """ + return self.condition(*tuple(assignment[v] for v in self.scope)) + + +def all_diff_constraint(*values): + """Returns True if all values are different, False otherwise""" + return len(values) is len(set(values)) + + +def is_word_constraint(words): + """Returns True if the letters concatenated form a word in words, False otherwise""" + + def isw(*letters): + return "".join(letters) in words + + return isw + + +def meet_at_constraint(p1, p2): + """Returns a function that is True when the words meet at the positions (p1, p2), False otherwise""" + + def meets(w1, w2): + return w1[p1] == w2[p2] + + meets.__name__ = "meet_at(" + str(p1) + ',' + str(p2) + ')' + return meets + + +def adjacent_constraint(x, y): + """Returns True if x and y are adjacent numbers, False otherwise""" + return abs(x - y) == 1 + + +def sum_constraint(n): + """Returns a function that is True when the the sum of all values is n, False otherwise""" + + def sumv(*values): + return sum(values) is n + + sumv.__name__ = str(n) + "==sum" + return sumv + + +def is_constraint(val): + """Returns a function that is True when x is equal to val, False otherwise""" + + def isv(x): + return val == x + + isv.__name__ = str(val) + "==" + return isv + + +def ne_constraint(val): + """Returns a function that is True when x is not equal to val, False otherwise""" + + def nev(x): + return val != x + + nev.__name__ = str(val) + "!=" + return nev + + +def no_heuristic(to_do): + """Return the to_do arc set unchanged (no ordering heuristic for GAC).""" + return to_do + + +def sat_up(to_do): + """Order the to_do arc set so that (variable, constraint) pairs whose constraint + has the smallest scope are processed first (a SortedSet keyed by 1 / scope size).""" + return SortedSet(to_do, key=lambda t: 1 / len([var for var in t[1].scope])) + + +class ACSolver: + """Solves a CSP with arc consistency and domain splitting""" + + def __init__(self, csp): + """a CSP solver that uses arc consistency + * csp is the CSP to be solved + """ + self.csp = csp + + def GAC(self, orig_domains=None, to_do=None, arc_heuristic=sat_up): + """ + Makes this CSP arc-consistent using Generalized Arc Consistency + orig_domains: is the original domains + to_do : is a set of (variable,constraint) pairs + returns the reduced domains (an arc-consistent variable:domain dictionary) + """ + if orig_domains is None: + orig_domains = self.csp.domains + if to_do is None: + to_do = {(var, const) for const in self.csp.constraints for var in const.scope} + else: + to_do = to_do.copy() + domains = orig_domains.copy() + to_do = arc_heuristic(to_do) + checks = 0 + while to_do: + var, const = to_do.pop() + other_vars = [ov for ov in const.scope if ov != var] + new_domain = set() + if len(other_vars) == 0: + for val in domains[var]: + if const.holds({var: val}): + new_domain.add(val) + checks += 1 + # new_domain = {val for val in domains[var] + # if const.holds({var: val})} + elif len(other_vars) == 1: + other = other_vars[0] + for val in domains[var]: + for other_val in domains[other]: + checks += 1 + if const.holds({var: val, other: other_val}): + new_domain.add(val) + break + # new_domain = {val for val in domains[var] + # if any(const.holds({var: val, other: other_val}) + # for other_val in domains[other])} + else: # general case + for val in domains[var]: + holds, checks = self.any_holds(domains, const, {var: val}, other_vars, checks=checks) + if holds: + new_domain.add(val) + # new_domain = {val for val in domains[var] + # if self.any_holds(domains, const, {var: val}, other_vars)} + if new_domain != domains[var]: + domains[var] = new_domain + if not new_domain: + return False, domains, checks + add_to_do = self.new_to_do(var, const).difference(to_do) + to_do |= add_to_do + return True, domains, checks + + def new_to_do(self, var, const): + """ + Returns new elements to be added to to_do after assigning + variable var in constraint const. + """ + return {(nvar, nconst) for nconst in self.csp.var_to_const[var] + if nconst != const + for nvar in nconst.scope + if nvar != var} + + def any_holds(self, domains, const, env, other_vars, ind=0, checks=0): + """ + Returns True if Constraint const holds for an assignment + that extends env with the variables in other_vars[ind:] + env is a dictionary + Warning: this has side effects and changes the elements of env + """ + if ind == len(other_vars): + return const.holds(env), checks + 1 + else: + var = other_vars[ind] + for val in domains[var]: + # env = dict_union(env, {var:val}) # no side effects + env[var] = val + holds, checks = self.any_holds(domains, const, env, other_vars, ind + 1, checks) + if holds: + return True, checks + return False, checks + + def domain_splitting(self, domains=None, to_do=None, arc_heuristic=sat_up): + """ + Return a solution to the current CSP or False if there are no solutions + to_do is the list of arcs to check + """ + if domains is None: + domains = self.csp.domains + consistency, new_domains, _ = self.GAC(domains, to_do, arc_heuristic) + if not consistency: + return False + elif all(len(new_domains[var]) == 1 for var in domains): + return {var: first(new_domains[var]) for var in domains} + else: + var = first(x for x in self.csp.variables if len(new_domains[x]) > 1) + if var: + dom1, dom2 = partition_domain(new_domains[var]) + new_doms1 = extend(new_domains, var, dom1) + new_doms2 = extend(new_domains, var, dom2) + to_do = self.new_to_do(var, None) + return self.domain_splitting(new_doms1, to_do, arc_heuristic) or \ + self.domain_splitting(new_doms2, to_do, arc_heuristic) + + +def partition_domain(dom): + """Partitions domain dom into two""" + split = len(dom) // 2 + dom1 = set(list(dom)[:split]) + dom2 = dom - dom1 + return dom1, dom2 + + +class ACSearchSolver(search.Problem): + """A search problem with arc consistency and domain splitting + A node is a CSP""" + + def __init__(self, csp, arc_heuristic=sat_up): + self.cons = ACSolver(csp) + consistency, self.domains, _ = self.cons.GAC(arc_heuristic=arc_heuristic) + if not consistency: + raise Exception('CSP is inconsistent') + self.heuristic = arc_heuristic + super().__init__(self.domains) + + def goal_test(self, node): + """Node is a goal if all domains have 1 element""" + return all(len(node[var]) == 1 for var in node) + + def actions(self, state): + """Return the successor states obtained by splitting some still-multi-valued + variable's domain in two and enforcing GAC on each half; only arc-consistent + results are kept.""" + var = first(x for x in state if len(state[x]) > 1) + neighs = [] + if var: + dom1, dom2 = partition_domain(state[var]) + to_do = self.cons.new_to_do(var, None) + for dom in [dom1, dom2]: + new_domains = extend(state, var, dom) + consistency, cons_doms, _ = self.cons.GAC(new_domains, to_do, self.heuristic) + if consistency: + neighs.append(cons_doms) + return neighs + + def result(self, state, action): + """Return the successor state, which is just the chosen reduced domains.""" + return action + + +def ac_solver(csp, arc_heuristic=sat_up): + """Arc consistency (domain splitting interface)""" + return ACSolver(csp).domain_splitting(arc_heuristic=arc_heuristic) + + +def ac_search_solver(csp, arc_heuristic=sat_up): + """Arc consistency (search interface)""" + from aima.search import depth_first_tree_search + solution = None + try: + solution = depth_first_tree_search(ACSearchSolver(csp, arc_heuristic=arc_heuristic)).state + except: + return solution + if solution: + return {var: first(solution[var]) for var in solution} + + +# ______________________________________________________________________________ +# Crossword Problem + + +csp_crossword = NaryCSP({'one_across': {'ant', 'big', 'bus', 'car', 'has'}, + 'one_down': {'book', 'buys', 'hold', 'lane', 'year'}, + 'two_down': {'ginger', 'search', 'symbol', 'syntax'}, + 'three_across': {'book', 'buys', 'hold', 'land', 'year'}, + 'four_across': {'ant', 'big', 'bus', 'car', 'has'}}, + [Constraint(('one_across', 'one_down'), meet_at_constraint(0, 0)), + Constraint(('one_across', 'two_down'), meet_at_constraint(2, 0)), + Constraint(('three_across', 'two_down'), meet_at_constraint(2, 2)), + Constraint(('three_across', 'one_down'), meet_at_constraint(0, 2)), + Constraint(('four_across', 'two_down'), meet_at_constraint(0, 4))]) + +crossword1 = [['_', '_', '_', '*', '*'], + ['_', '*', '_', '*', '*'], + ['_', '_', '_', '_', '*'], + ['_', '*', '_', '*', '*'], + ['*', '*', '_', '_', '_'], + ['*', '*', '_', '*', '*']] + +words1 = {'ant', 'big', 'bus', 'car', 'has', 'book', 'buys', 'hold', + 'lane', 'year', 'ginger', 'search', 'symbol', 'syntax'} + + +class Crossword(NaryCSP): + """An n-ary CSP for a crossword puzzle. The puzzle is a grid where '_' marks a + writable cell and '*' a blocked cell; each maximal run of two or more cells (across + or down) becomes a variable sequence constrained to spell one of the given words.""" + + def __init__(self, puzzle, words): + domains = {} + constraints = [] + for i, line in enumerate(puzzle): + scope = [] + for j, element in enumerate(line): + if element == '_': + var = "p" + str(j) + str(i) + domains[var] = list(string.ascii_lowercase) + scope.append(var) + else: + if len(scope) > 1: + constraints.append(Constraint(tuple(scope), is_word_constraint(words))) + scope.clear() + if len(scope) > 1: + constraints.append(Constraint(tuple(scope), is_word_constraint(words))) + puzzle_t = list(map(list, zip(*puzzle))) + for i, line in enumerate(puzzle_t): + scope = [] + for j, element in enumerate(line): + if element == '_': + scope.append("p" + str(i) + str(j)) + else: + if len(scope) > 1: + constraints.append(Constraint(tuple(scope), is_word_constraint(words))) + scope.clear() + if len(scope) > 1: + constraints.append(Constraint(tuple(scope), is_word_constraint(words))) + super().__init__(domains, constraints) + self.puzzle = puzzle + + def display(self, assignment=None): + """Print the crossword grid, filling in letters from assignment where known.""" + for i, line in enumerate(self.puzzle): + puzzle = "" + for j, element in enumerate(line): + if element == '*': + puzzle += "[*] " + else: + var = "p" + str(j) + str(i) + if assignment is not None: + if isinstance(assignment[var], set) and len(assignment[var]) == 1: + puzzle += "[" + str(first(assignment[var])).upper() + "] " + elif isinstance(assignment[var], str): + puzzle += "[" + str(assignment[var]).upper() + "] " + else: + puzzle += "[_] " + else: + puzzle += "[_] " + print(puzzle) + + +# ______________________________________________________________________________ +# Kakuro Problem + + +# difficulty 0 +kakuro1 = [['*', '*', '*', [6, ''], [3, '']], + ['*', [4, ''], [3, 3], '_', '_'], + [['', 10], '_', '_', '_', '_'], + [['', 3], '_', '_', '*', '*']] + +# difficulty 0 +kakuro2 = [ + ['*', [10, ''], [13, ''], '*'], + [['', 3], '_', '_', [13, '']], + [['', 12], '_', '_', '_'], + [['', 21], '_', '_', '_']] + +# difficulty 1 +kakuro3 = [ + ['*', [17, ''], [28, ''], '*', [42, ''], [22, '']], + [['', 9], '_', '_', [31, 14], '_', '_'], + [['', 20], '_', '_', '_', '_', '_'], + ['*', ['', 30], '_', '_', '_', '_'], + ['*', [22, 24], '_', '_', '_', '*'], + [['', 25], '_', '_', '_', '_', [11, '']], + [['', 20], '_', '_', '_', '_', '_'], + [['', 14], '_', '_', ['', 17], '_', '_']] + +# difficulty 2 +kakuro4 = [ + ['*', '*', '*', '*', '*', [4, ''], [24, ''], [11, ''], '*', '*', '*', [11, ''], [17, ''], '*', '*'], + ['*', '*', '*', [17, ''], [11, 12], '_', '_', '_', '*', '*', [24, 10], '_', '_', [11, ''], '*'], + ['*', [4, ''], [16, 26], '_', '_', '_', '_', '_', '*', ['', 20], '_', '_', '_', '_', [16, '']], + [['', 20], '_', '_', '_', '_', [24, 13], '_', '_', [16, ''], ['', 12], '_', '_', [23, 10], '_', '_'], + [['', 10], '_', '_', [24, 12], '_', '_', [16, 5], '_', '_', [16, 30], '_', '_', '_', '_', '_'], + ['*', '*', [3, 26], '_', '_', '_', '_', ['', 12], '_', '_', [4, ''], [16, 14], '_', '_', '*'], + ['*', ['', 8], '_', '_', ['', 15], '_', '_', [34, 26], '_', '_', '_', '_', '_', '*', '*'], + ['*', ['', 11], '_', '_', [3, ''], [17, ''], ['', 14], '_', '_', ['', 8], '_', '_', [7, ''], [17, ''], '*'], + ['*', '*', '*', [23, 10], '_', '_', [3, 9], '_', '_', [4, ''], [23, ''], ['', 13], '_', '_', '*'], + ['*', '*', [10, 26], '_', '_', '_', '_', '_', ['', 7], '_', '_', [30, 9], '_', '_', '*'], + ['*', [17, 11], '_', '_', [11, ''], [24, 8], '_', '_', [11, 21], '_', '_', '_', '_', [16, ''], [17, '']], + [['', 29], '_', '_', '_', '_', '_', ['', 7], '_', '_', [23, 14], '_', '_', [3, 17], '_', '_'], + [['', 10], '_', '_', [3, 10], '_', '_', '*', ['', 8], '_', '_', [4, 25], '_', '_', '_', '_'], + ['*', ['', 16], '_', '_', '_', '_', '*', ['', 23], '_', '_', '_', '_', '_', '*', '*'], + ['*', '*', ['', 6], '_', '_', '*', '*', ['', 15], '_', '_', '_', '*', '*', '*', '*']] + + +class Kakuro(NaryCSP): + """An n-ary CSP for a Kakuro puzzle. Each '_' cell is a variable with domain 1..9; + every horizontal or vertical run of cells must contain distinct digits that sum to + the clue given in the adjacent '[down, right]' header cell.""" + + def __init__(self, puzzle): + variables = [] + for i, line in enumerate(puzzle): + # print line + for j, element in enumerate(line): + if element == '_': + var1 = str(i) + if len(var1) == 1: + var1 = "0" + var1 + var2 = str(j) + if len(var2) == 1: + var2 = "0" + var2 + variables.append("X" + var1 + var2) + domains = {} + for var in variables: + domains[var] = set(range(1, 10)) + constraints = [] + for i, line in enumerate(puzzle): + for j, element in enumerate(line): + if element != '_' and element != '*': + # down - column + if element[0] != '': + x = [] + for k in range(i + 1, len(puzzle)): + if puzzle[k][j] != '_': + break + var1 = str(k) + if len(var1) == 1: + var1 = "0" + var1 + var2 = str(j) + if len(var2) == 1: + var2 = "0" + var2 + x.append("X" + var1 + var2) + constraints.append(Constraint(x, sum_constraint(element[0]))) + constraints.append(Constraint(x, all_diff_constraint)) + # right - line + if element[1] != '': + x = [] + for k in range(j + 1, len(puzzle[i])): + if puzzle[i][k] != '_': + break + var1 = str(i) + if len(var1) == 1: + var1 = "0" + var1 + var2 = str(k) + if len(var2) == 1: + var2 = "0" + var2 + x.append("X" + var1 + var2) + constraints.append(Constraint(x, sum_constraint(element[1]))) + constraints.append(Constraint(x, all_diff_constraint)) + super().__init__(domains, constraints) + self.puzzle = puzzle + + def display(self, assignment=None): + """Print the Kakuro grid, showing clue cells and filling in digits from + assignment where known.""" + for i, line in enumerate(self.puzzle): + puzzle = "" + for j, element in enumerate(line): + if element == '*': + puzzle += "[*]\t" + elif element == '_': + var1 = str(i) + if len(var1) == 1: + var1 = "0" + var1 + var2 = str(j) + if len(var2) == 1: + var2 = "0" + var2 + var = "X" + var1 + var2 + if assignment is not None: + if isinstance(assignment[var], set) and len(assignment[var]) == 1: + puzzle += "[" + str(first(assignment[var])) + "]\t" + elif isinstance(assignment[var], int): + puzzle += "[" + str(assignment[var]) + "]\t" + else: + puzzle += "[_]\t" + else: + puzzle += "[_]\t" + else: + puzzle += str(element[0]) + "\\" + str(element[1]) + "\t" + print(puzzle) + + +# ______________________________________________________________________________ +# Cryptarithmetic Problem + +# [Figure 6.2] +# T W O + T W O = F O U R +two_two_four = NaryCSP({'T': set(range(1, 10)), 'F': set(range(1, 10)), + 'W': set(range(0, 10)), 'O': set(range(0, 10)), 'U': set(range(0, 10)), 'R': set(range(0, 10)), + 'C1': set(range(0, 2)), 'C2': set(range(0, 2)), 'C3': set(range(0, 2))}, + [Constraint(('T', 'F', 'W', 'O', 'U', 'R'), all_diff_constraint), + Constraint(('O', 'R', 'C1'), lambda o, r, c1: o + o == r + 10 * c1), + Constraint(('W', 'U', 'C1', 'C2'), lambda w, u, c1, c2: c1 + w + w == u + 10 * c2), + Constraint(('T', 'O', 'C2', 'C3'), lambda t, o, c2, c3: c2 + t + t == o + 10 * c3), + Constraint(('F', 'C3'), eq)]) + +# S E N D + M O R E = M O N E Y +send_more_money = NaryCSP({'S': set(range(1, 10)), 'M': set(range(1, 10)), + 'E': set(range(0, 10)), 'N': set(range(0, 10)), 'D': set(range(0, 10)), + 'O': set(range(0, 10)), 'R': set(range(0, 10)), 'Y': set(range(0, 10)), + 'C1': set(range(0, 2)), 'C2': set(range(0, 2)), 'C3': set(range(0, 2)), + 'C4': set(range(0, 2))}, + [Constraint(('S', 'E', 'N', 'D', 'M', 'O', 'R', 'Y'), all_diff_constraint), + Constraint(('D', 'E', 'Y', 'C1'), lambda d, e, y, c1: d + e == y + 10 * c1), + Constraint(('N', 'R', 'E', 'C1', 'C2'), lambda n, r, e, c1, c2: c1 + n + r == e + 10 * c2), + Constraint(('E', 'O', 'N', 'C2', 'C3'), lambda e, o, n, c2, c3: c2 + e + o == n + 10 * c3), + Constraint(('S', 'M', 'O', 'C3', 'C4'), lambda s, m, o, c3, c4: c3 + s + m == o + 10 * c4), + Constraint(('M', 'C4'), eq)]) diff --git a/q-learning-9b/aima/deep_learning.py b/q-learning-9b/aima/deep_learning.py new file mode 100644 index 0000000..7c69da9 --- /dev/null +++ b/q-learning-9b/aima/deep_learning.py @@ -0,0 +1,658 @@ +"""Deep learning. (Chapters 20)""" + +import random +import statistics + +import numpy as np +from keras import Sequential, optimizers +from keras.layers import Embedding, SimpleRNN, Dense +from keras.preprocessing import sequence + +from aima.utils import conv1D, gaussian_kernel, random_weights, map_vector, mean_squared_error_loss + + +# The neural-network code operates on nested weight matrices, so it needs the +# recursive variants of these vector helpers (the canonical aima.utils versions +# are the flat ones used by the grid/search code, which return tuples/np arrays). +def element_wise_product(x, y): + """Element-wise product of x and y, recursing into nested iterables.""" + if hasattr(x, '__iter__') and hasattr(y, '__iter__'): + assert len(x) == len(y) + return [element_wise_product(_x, _y) for _x, _y in zip(x, y)] + elif hasattr(x, '__iter__') == hasattr(y, '__iter__'): + return x * y + else: + raise Exception('Inputs must be in the same size!') + + +def vector_add(a, b): + """Component-wise (recursive) addition of two vectors.""" + if not (a and b): + return a or b + if hasattr(a, '__iter__') and hasattr(b, '__iter__'): + assert len(a) == len(b) + return list(map(vector_add, a, b)) + else: + try: + return a + b + except TypeError: + raise Exception('Inputs must be in the same size!') + + +def scalar_vector_product(x, y): + """Product of a scalar x and a (possibly nested) vector y, recursively.""" + return [scalar_vector_product(x, _y) for _y in y] if hasattr(y, '__iter__') else x * y + + +class Node: + """ + A single unit of a layer in a neural network + :param weights: weights between parent nodes and current node + :param value: value of current node + """ + + def __init__(self, weights=None, value=None): + self.value = value + self.weights = weights or [] + + +class Layer: + """ + A layer in a neural network based on a computational graph. + :param size: number of units in the current layer + """ + + def __init__(self, size): + self.nodes = np.array([Node() for _ in range(size)]) + + def forward(self, inputs): + """Define the operation to get the output of this layer""" + raise NotImplementedError + + +class Activation: + """Abstract base class for neural-network activation functions. + + Subclasses implement ``function`` and its ``derivative``; calling an + instance applies the activation to its input. + """ + + def function(self, x): + """Apply the activation function to input ``x``.""" + raise NotImplementedError + + def derivative(self, x): + """Return the derivative of the activation function at ``x``.""" + raise NotImplementedError + + def __call__(self, x): + return self.function(x) + + +class Sigmoid(Activation): + """Logistic sigmoid activation, ``1 / (1 + e**-x)``.""" + + def function(self, x): + """Return the logistic sigmoid of ``x``.""" + return 1 / (1 + np.exp(-x)) + + def derivative(self, value): + """Return the sigmoid derivative given the layer output ``value``.""" + return value * (1 - value) + + +class ReLU(Activation): + """Rectified Linear Unit activation, ``max(0, x)``.""" + + def function(self, x): + """Return ``max(0, x)``.""" + return max(0, x) + + def derivative(self, value): + """Return the ReLU derivative (1 if ``value`` > 0 else 0).""" + return 1 if value > 0 else 0 + + +class ELU(Activation): + """Exponential Linear Unit activation, with scale ``alpha`` for non-positive inputs.""" + + def __init__(self, alpha=0.01): + self.alpha = alpha + + def function(self, x): + """Return ``x`` if positive else ``alpha * (e**x - 1)``.""" + return x if x > 0 else self.alpha * (np.exp(x) - 1) + + def derivative(self, value): + """Return the ELU derivative given the layer output ``value``.""" + return 1 if value > 0 else self.alpha * np.exp(value) + + +class LeakyReLU(Activation): + """Leaky ReLU activation, with small slope ``alpha`` for negative inputs.""" + + def __init__(self, alpha=0.01): + self.alpha = alpha + + def function(self, x): + """Return ``max(x, alpha * x)``.""" + return max(x, self.alpha * x) + + def derivative(self, value): + """Return the Leaky ReLU derivative (1 if ``value`` > 0 else ``alpha``).""" + return 1 if value > 0 else self.alpha + + +class Tanh(Activation): + """Hyperbolic tangent activation.""" + + def function(self, x): + """Return ``tanh(x)``.""" + return np.tanh(x) + + def derivative(self, value): + """Return the tanh derivative given the layer output ``value`` (``1 - value**2``).""" + return 1 - (value ** 2) + + +class SoftMax(Activation): + """Softmax activation that normalises a vector into a probability distribution.""" + + def function(self, x): + """Return the softmax of vector ``x`` (normalised exponentials).""" + return np.exp(x) / np.sum(np.exp(x)) + + def derivative(self, x): + """Return a placeholder unit gradient for each element of ``x``.""" + return np.ones_like(x) + + +class SoftPlus(Activation): + """SoftPlus activation, ``log(1 + e**x)`` (a smooth approximation of ReLU).""" + + def function(self, x): + """Return ``log(1 + e**x)`` for ``x``.""" + return np.log(1. + np.exp(x)) + + def derivative(self, x): + """Return the SoftPlus derivative at ``x`` (the logistic sigmoid).""" + return 1. / (1. + np.exp(-x)) + + +class Linear(Activation): + """Identity (linear) activation that returns its input unchanged.""" + + def function(self, x): + """Return ``x`` unchanged.""" + return x + + def derivative(self, x): + """Return an all-ones gradient matching the shape of ``x``.""" + return np.ones_like(x) + + +class InputLayer(Layer): + """1D input layer. Layer size is the same as input vector size.""" + + def __init__(self, size=3): + super().__init__(size) + + def forward(self, inputs): + """Take each value of the inputs to each unit in the layer.""" + assert len(self.nodes) == len(inputs) + for node, inp in zip(self.nodes, inputs): + node.value = inp + return inputs + + +class OutputLayer(Layer): + """1D softmax output layer in 19.3.2.""" + + def __init__(self, size=3): + super().__init__(size) + + def forward(self, inputs, activation=SoftMax): + """Apply ``activation`` (softmax by default) to ``inputs`` and store it in each node.""" + assert len(self.nodes) == len(inputs) + res = activation().function(inputs) + for node, val in zip(self.nodes, res): + node.value = val + return res + + +class DenseLayer(Layer): + """ + 1D dense layer in a neural network. + :param in_size: (int) input vector size + :param out_size: (int) output vector size + :param activation: (Activation object) activation function + """ + + def __init__(self, in_size=3, out_size=3, activation=Sigmoid): + super().__init__(out_size) + self.out_size = out_size + self.inputs = None + self.activation = activation() + # initialize weights + for node in self.nodes: + node.weights = random_weights(-0.5, 0.5, in_size) + + def forward(self, inputs): + """Apply the activation to each unit's weighted sum of ``inputs`` and return the outputs.""" + self.inputs = inputs + res = [] + # get the output value of each unit + for unit in self.nodes: + val = self.activation.function(np.dot(unit.weights, inputs)) + unit.value = val + res.append(val) + return res + + +class ConvLayer1D(Layer): + """ + 1D convolution layer of in neural network. + :param kernel_size: convolution kernel size + """ + + def __init__(self, size=3, kernel_size=3): + super().__init__(size) + # init convolution kernel as gaussian kernel + for node in self.nodes: + node.weights = gaussian_kernel(kernel_size) + + def forward(self, features): + """Convolve each input channel in ``features`` with its node kernel and return the outputs.""" + # each node in layer takes a channel in the features + assert len(self.nodes) == len(features) + res = [] + # compute the convolution output of each channel, store it in node.val + for node, feature in zip(self.nodes, features): + out = conv1D(feature, node.weights) + res.append(out) + node.value = out + return res + + +class MaxPoolingLayer1D(Layer): + """ + 1D max pooling layer in a neural network. + :param kernel_size: max pooling area size + """ + + def __init__(self, size=3, kernel_size=3): + super().__init__(size) + self.kernel_size = kernel_size + self.inputs = None + + def forward(self, features): + """Apply 1D max pooling over each channel in ``features`` and return the pooled outputs.""" + assert len(self.nodes) == len(features) + res = [] + self.inputs = features + # do max pooling for each channel in features + for i in range(len(self.nodes)): + feature = features[i] + # get the max value in a kernel_size * kernel_size area + out = [max(feature[i:i + self.kernel_size]) + for i in range(len(feature) - self.kernel_size + 1)] + res.append(out) + self.nodes[i].value = out + return res + + +class BatchNormalizationLayer(Layer): + """Batch normalization layer.""" + + def __init__(self, size, eps=0.001): + super().__init__(size) + self.eps = eps + # self.weights = [beta, gamma] + self.weights = [0, 0] + self.inputs = None + + def forward(self, inputs): + """Normalise ``inputs`` by their mean and std, then scale and shift by the layer weights.""" + # mean value of inputs + mu = sum(inputs) / len(inputs) + # standard error of inputs + stderr = statistics.stdev(inputs) + self.inputs = inputs + res = [] + # get normalized value of each input + for i in range(len(self.nodes)): + val = [(inputs[i] - mu) * self.weights[0] / np.sqrt(self.eps + stderr ** 2) + self.weights[1]] + res.append(val) + self.nodes[i].value = val + return res + + +def init_examples(examples, idx_i, idx_t, o_units): + """Init examples from dataset.examples.""" + + inputs, targets = {}, {} + for i, e in enumerate(examples): + # input values of e + inputs[i] = [e[i] for i in idx_i] + + if o_units > 1: + # one-hot representation of e's target + t = [0 for i in range(o_units)] + t[e[idx_t]] = 1 + targets[i] = t + else: + # target value of e + targets[i] = [e[idx_t]] + + return inputs, targets + + +def stochastic_gradient_descent(dataset, net, loss, epochs=1000, l_rate=0.01, batch_size=1, verbose=False): + """ + Gradient descent algorithm to update the learnable parameters of a network. + :return: the updated network + """ + examples = dataset.examples # init data + + for e in range(epochs): + total_loss = 0 + random.shuffle(examples) + weights = [[node.weights for node in layer.nodes] for layer in net] + + for batch in get_batch(examples, batch_size): + inputs, targets = init_examples(batch, dataset.inputs, dataset.target, len(net[-1].nodes)) + # compute gradients of weights + gs, batch_loss = BackPropagation(inputs, targets, weights, net, loss) + # update weights with gradient descent + weights = [x + y for x, y in zip(weights, [np.array(tg) * -l_rate for tg in gs])] + total_loss += batch_loss + + # update the weights of network each batch + for i in range(len(net)): + if weights[i].size != 0: + for j in range(len(weights[i])): + net[i].nodes[j].weights = weights[i][j] + + if verbose: + print("epoch:{}, total_loss:{}".format(e + 1, total_loss)) + + return net + + +def adam(dataset, net, loss, epochs=1000, rho=(0.9, 0.999), delta=1 / 10 ** 8, + l_rate=0.001, batch_size=1, verbose=False): + """ + [Figure 19.6] + Adam optimizer to update the learnable parameters of a network. + Required parameters are similar to gradient descent. + :return the updated network + """ + examples = dataset.examples + + # init s,r and t + s = [[[0] * len(node.weights) for node in layer.nodes] for layer in net] + r = [[[0] * len(node.weights) for node in layer.nodes] for layer in net] + t = 0 + + # repeat util converge + for e in range(epochs): + # total loss of each epoch + total_loss = 0 + random.shuffle(examples) + weights = [[node.weights for node in layer.nodes] for layer in net] + + for batch in get_batch(examples, batch_size): + t += 1 + inputs, targets = init_examples(batch, dataset.inputs, dataset.target, len(net[-1].nodes)) + + # compute gradients of weights + gs, batch_loss = BackPropagation(inputs, targets, weights, net, loss) + + # update s,r,s_hat and r_gat + s = vector_add(scalar_vector_product(rho[0], s), + scalar_vector_product((1 - rho[0]), gs)) + r = vector_add(scalar_vector_product(rho[1], r), + scalar_vector_product((1 - rho[1]), element_wise_product(gs, gs))) + s_hat = scalar_vector_product(1 / (1 - rho[0] ** t), s) + r_hat = scalar_vector_product(1 / (1 - rho[1] ** t), r) + + # rescale r_hat + r_hat = map_vector(lambda x: 1 / (np.sqrt(x) + delta), r_hat) + + # delta weights + delta_theta = scalar_vector_product(-l_rate, element_wise_product(s_hat, r_hat)) + weights = vector_add(weights, delta_theta) + total_loss += batch_loss + + # update the weights of network each batch + for i in range(len(net)): + if weights[i]: + for j in range(len(weights[i])): + net[i].nodes[j].weights = weights[i][j] + + if verbose: + print("epoch:{}, total_loss:{}".format(e + 1, total_loss)) + + return net + + +def BackPropagation(inputs, targets, theta, net, loss): + """ + The back-propagation algorithm for multilayer networks in only one epoch, to calculate gradients of theta. + :param inputs: a batch of inputs in an array. Each input is an iterable object + :param targets: a batch of targets in an array. Each target is an iterable object + :param theta: parameters to be updated + :param net: a list of predefined layer objects representing their linear sequence + :param loss: a predefined loss function taking array of inputs and targets + :return: gradients of theta, loss of the input batch + """ + + assert len(inputs) == len(targets) + o_units = len(net[-1].nodes) + n_layers = len(net) + batch_size = len(inputs) + + gradients = [[[] for _ in layer.nodes] for layer in net] + total_gradients = [[[0] * len(node.weights) for node in layer.nodes] for layer in net] + + batch_loss = 0 + + # iterate over each example in batch + for e in range(batch_size): + i_val = inputs[e] + t_val = targets[e] + + # forward pass and compute batch loss + for i in range(1, n_layers): + layer_out = net[i].forward(i_val) + i_val = layer_out + batch_loss += loss(t_val, layer_out) + + # initialize delta + delta = [[] for _ in range(n_layers)] + + previous = np.array([layer_out[i] - t_val[i] for i in range(o_units)]) + h_layers = n_layers - 1 + + # backward pass + for i in range(h_layers, 0, -1): + layer = net[i] + derivative = np.array([layer.activation.derivative(node.value) for node in layer.nodes]) + delta[i] = previous * derivative + # pass to layer i-1 in the next iteration + previous = np.matmul([delta[i]], theta[i])[0] + # compute gradient of layer i + gradients[i] = [scalar_vector_product(d, net[i].inputs) for d in delta[i]] + + # add gradient of current example to batch gradient + total_gradients = vector_add(total_gradients, gradients) + + return total_gradients, batch_loss + + +def get_batch(examples, batch_size=1): + """Split examples into multiple batches""" + for i in range(0, len(examples), batch_size): + yield examples[i: i + batch_size] + + +class NeuralNetworkLearner: + """ + Simple dense multilayer neural network. + :param hidden_layer_sizes: size of hidden layers in the form of a list + """ + + def __init__(self, dataset, hidden_layer_sizes, l_rate=0.01, epochs=1000, batch_size=10, + optimizer=stochastic_gradient_descent, loss=mean_squared_error_loss, verbose=False, plot=False): + self.dataset = dataset + self.l_rate = l_rate + self.epochs = epochs + self.batch_size = batch_size + self.optimizer = optimizer + self.loss = loss + self.verbose = verbose + self.plot = plot + + input_size = len(dataset.inputs) + output_size = len(dataset.values[dataset.target]) + + # initialize the network + raw_net = [InputLayer(input_size)] + # add hidden layers + hidden_input_size = input_size + for h_size in hidden_layer_sizes: + raw_net.append(DenseLayer(hidden_input_size, h_size)) + hidden_input_size = h_size + raw_net.append(DenseLayer(hidden_input_size, output_size)) + self.raw_net = raw_net + + def fit(self, X, y): + """Train the network with the configured optimizer and loss, returning ``self``.""" + self.learned_net = self.optimizer(self.dataset, self.raw_net, loss=self.loss, epochs=self.epochs, + l_rate=self.l_rate, batch_size=self.batch_size, verbose=self.verbose) + return self + + def predict(self, example): + """Forward-pass ``example`` through the trained net and return the index of the max output.""" + n_layers = len(self.learned_net) + + layer_input = example + layer_out = example + + # get the output of each layer by forward passing + for i in range(1, n_layers): + layer_out = self.learned_net[i].forward(np.array(layer_input).reshape((-1, 1))) + layer_input = layer_out + + return layer_out.index(max(layer_out)) + + +class PerceptronLearner: + """ + Simple perceptron neural network. + """ + + def __init__(self, dataset, l_rate=0.01, epochs=1000, batch_size=10, optimizer=stochastic_gradient_descent, + loss=mean_squared_error_loss, verbose=False, plot=False): + self.dataset = dataset + self.l_rate = l_rate + self.epochs = epochs + self.batch_size = batch_size + self.optimizer = optimizer + self.loss = loss + self.verbose = verbose + self.plot = plot + + input_size = len(dataset.inputs) + output_size = len(dataset.values[dataset.target]) + + # initialize the network, add dense layer + self.raw_net = [InputLayer(input_size), DenseLayer(input_size, output_size)] + + def fit(self, X, y): + """Train the perceptron with the configured optimizer and loss, returning ``self``.""" + self.learned_net = self.optimizer(self.dataset, self.raw_net, loss=self.loss, epochs=self.epochs, + l_rate=self.l_rate, batch_size=self.batch_size, verbose=self.verbose) + return self + + def predict(self, example): + """Forward-pass ``example`` and return the index of the maximum output unit.""" + layer_out = self.learned_net[1].forward(np.array(example).reshape((-1, 1))) + return layer_out.index(max(layer_out)) + + +def keras_dataset_loader(dataset, max_length=500): + """ + Helper function to load keras datasets. + :param dataset: keras data set type + :param max_length: max length of each input sequence + """ + # init dataset + (X_train, y_train), (X_val, y_val) = dataset + if max_length > 0: + X_train = sequence.pad_sequences(X_train, maxlen=max_length) + X_val = sequence.pad_sequences(X_val, maxlen=max_length) + return (X_train[10:], y_train[10:]), (X_val, y_val), (X_train[:10], y_train[:10]) + + +def SimpleRNNLearner(train_data, val_data, epochs=2, verbose=False): + """ + RNN example for text sentimental analysis. + + :param train_data: + a tuple of (training data, targets) + Training data: ndarray taking training examples, while each example is coded by embedding + Targets: ndarray taking targets of each example. Each target is mapped to an integer + :param val_data: a tuple of (validation data, targets) + :param epochs: number of epochs + :param verbose: verbosity mode + :return: a keras model + """ + + total_inputs = 5000 + input_length = 500 + + # init data + X_train, y_train = train_data + X_val, y_val = val_data + + # init a the sequential network (embedding layer, rnn layer, dense layer) + model = Sequential() + model.add(Embedding(total_inputs, 32, input_length=input_length)) + model.add(SimpleRNN(units=128)) + model.add(Dense(1, activation='sigmoid')) + model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) + + # train the model + model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=epochs, batch_size=128, verbose=verbose) + + return model + + +def AutoencoderLearner(inputs, encoding_size, epochs=200, verbose=False): + """ + Simple example of linear auto encoder learning producing the input itself. + :param inputs: a batch of input data in np.ndarray type + :param encoding_size: int, the size of encoding layer + :param epochs: number of epochs + :param verbose: verbosity mode + :return: a keras model + """ + + # init data + input_size = len(inputs[0]) + + # init model + model = Sequential() + model.add(Dense(encoding_size, input_dim=input_size, activation='relu', kernel_initializer='random_uniform', + bias_initializer='ones')) + model.add(Dense(input_size, activation='relu', kernel_initializer='random_uniform', bias_initializer='ones')) + + # update model with sgd + sgd = optimizers.SGD(learning_rate=0.01) + model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy']) + + # train the model + model.fit(inputs, inputs, epochs=epochs, batch_size=10, verbose=verbose) + + return model diff --git a/q-learning-9b/aima/game_theory.py b/q-learning-9b/aima/game_theory.py new file mode 100644 index 0000000..b25442f --- /dev/null +++ b/q-learning-9b/aima/game_theory.py @@ -0,0 +1,257 @@ +"""Multiagent decision making: game theory and social choice (Chapter 18).""" + +from collections import Counter, defaultdict +from itertools import combinations, permutations +from math import factorial + +import numpy as np +from scipy.optimize import linprog + + +def dominates(payoff, s, t, strongly=True): + """ + [Section 18.2] + True if, for the player whose payoff matrix is 'payoff' (rows = that player's + own strategies, columns = the opponents' joint strategies), strategy s + dominates strategy t. Strong domination requires s to be strictly better than + t against every opponent strategy; weak domination requires s to be no worse + everywhere and strictly better in at least one column. + """ + payoff = np.asarray(payoff, dtype=float) + if strongly: + return bool(np.all(payoff[s] > payoff[t])) + return bool(np.all(payoff[s] >= payoff[t]) and np.any(payoff[s] > payoff[t])) + + +def dominant_strategy(payoff, strongly=True): + """ + [Section 18.2] + Return the strategy (row index) that dominates all of the player's other + strategies, or None if the player has no such dominant strategy. To analyze + the column player of a game, pass the transpose of their payoff matrix so + that their strategies become the rows. + """ + payoff = np.asarray(payoff, dtype=float) + for s in range(payoff.shape[0]): + if all(dominates(payoff, s, t, strongly) for t in range(payoff.shape[0]) if t != s): + return s + return None + + +def iterated_dominance(payoff1, payoff2, strongly=True): + """ + [Section 18.2] + Iterated elimination of dominated strategies: since a rational player never + plays a dominated strategy, we can repeatedly delete, for either player, any + strategy that is dominated among the strategies still in play, until none + remains. 'payoff1'/'payoff2' are the row and column player's payoff matrices. + Returns the surviving (rows, cols) strategy indices of the original game. + """ + A, B = np.asarray(payoff1, dtype=float), np.asarray(payoff2, dtype=float) + rows, cols = list(range(A.shape[0])), list(range(A.shape[1])) + + def find_dominated(payoff, own, opponent): + """A strategy in 'own' that is dominated against the surviving 'opponent' strategies.""" + for s in own: + for t in own: + if s != t: + better, worse = payoff[t][opponent], payoff[s][opponent] + if (np.all(better > worse) if strongly else + np.all(better >= worse) and np.any(better > worse)): + return s + return None + + eliminated = True + while eliminated: + # the row player's payoff is A[row][col]; the column player's is B.T[col][row] + row = find_dominated(A, rows, cols) + if row is not None: + rows.remove(row) + col = find_dominated(B.T, cols, rows) + if col is not None: + cols.remove(col) + eliminated = row is not None or col is not None + + return rows, cols + + +def pure_nash_equilibria(payoff1, payoff2): + """ + [Section 18.2] + All pure-strategy Nash equilibria of a two-player game, given the payoff + matrices payoff1[i][j] and payoff2[i][j] for the row and column player when + the row player plays i and the column player plays j. A profile (i, j) is a + Nash equilibrium when each player is playing a best response to the other, so + neither can gain by deviating unilaterally. Returns a list of (i, j) profiles + (possibly empty, e.g. for matching pennies). + """ + payoff1, payoff2 = np.asarray(payoff1, dtype=float), np.asarray(payoff2, dtype=float) + m, n = payoff1.shape + return [(i, j) for i in range(m) for j in range(n) + # row player best-responds within column j and column player within row i + if payoff1[i, j] >= payoff1[:, j].max() and payoff2[i, j] >= payoff2[i, :].max()] + + +def solve_zero_sum_game(payoff): + """ + [Section 18.2] + Solve a two-player zero-sum game by linear programming. 'payoff' is the payoff + to the row (maximizing) player; the column (minimizing) player receives its + negation. By von Neumann's minimax theorem the game has a value v and optimal + mixed strategies x, y such that x maximizes the row player's guaranteed payoff + (for every column j, sum_i x_i payoff[i][j] >= v) and y minimizes the column + player's guaranteed loss. Returns (value, row_strategy, col_strategy). + """ + payoff = np.asarray(payoff, dtype=float) + m, n = payoff.shape + + # row player: maximize v s.t. for every column j, sum_i x_i payoff[i][j] >= v; + # variables are [x_1, ..., x_m, v], and linprog minimizes, so we minimize -v + res = linprog(c=np.append(np.zeros(m), -1), + A_ub=np.column_stack([-payoff.T, np.ones(n)]), b_ub=np.zeros(n), + A_eq=np.append(np.ones(m), 0).reshape(1, -1), b_eq=[1], + bounds=[(0, None)] * m + [(None, None)]) + value, row_strategy = res.x[m], res.x[:m] + + # column player: minimize w s.t. for every row i, sum_j payoff[i][j] y_j <= w + res = linprog(c=np.append(np.zeros(n), 1), + A_ub=np.column_stack([payoff, -np.ones(m)]), b_ub=np.zeros(m), + A_eq=np.append(np.ones(n), 0).reshape(1, -1), b_eq=[1], + bounds=[(0, None)] * n + [(None, None)]) + col_strategy = res.x[:n] + + return value, row_strategy, col_strategy + + +# ______________________________________________________________________________ +# 18.3 Cooperative Game Theory + + +def shapley_value(players, characteristic_function): + """ + [Section 18.3] + The Shapley value of a cooperative game (players, v): a fair division of the + grand coalition's value v(N) that pays each player the average, over all n! + orderings of the players, of the marginal contribution + mc_i(C) = v(C and {i}) - v(C) they make to the players preceding them. The + 'characteristic_function' is a callable mapping a frozenset of players to its + value. Returns a dict mapping each player to their Shapley value. + """ + players = list(players) + phi = {i: 0.0 for i in players} + for order in permutations(players): + preceding = set() + for i in order: + phi[i] += (characteristic_function(frozenset(preceding | {i})) + - characteristic_function(frozenset(preceding))) + preceding.add(i) + return {i: phi[i] / factorial(len(players)) for i in players} + + +def is_in_core(players, characteristic_function, payoff): + """ + [Section 18.3] + True if 'payoff' (a dict mapping each player to their share) lies in the core + of the cooperative game (players, v): it must distribute exactly the grand + coalition's value (sum of shares = v(N)) and be immune to defection, i.e. for + every coalition C the players in C receive at least v(C) (otherwise C would be + better off on its own). An empty core means the grand coalition cannot form. + """ + players, v = list(players), characteristic_function + # efficiency: the grand coalition's value is fully distributed + if not np.isclose(sum(payoff[i] for i in players), v(frozenset(players))): + return False + # no coalition can object: x(C) >= v(C) for every coalition C + return all(sum(payoff[i] for i in coalition) >= v(frozenset(coalition)) - 1e-9 + for size in range(1, len(players)) for coalition in combinations(players, size)) + + +# ______________________________________________________________________________ +# 18.4 Making Collective Decisions + + +def plurality_winner(preferences): + """ + [Section 18.4] + Winner under plurality voting: the candidate ranked first by the most voters. + 'preferences' is a list of ballots, each a list of candidates ordered from + most to least preferred. + """ + first_choices = Counter(ballot[0] for ballot in preferences) + return max(first_choices, key=first_choices.get) + + +def borda_winner(preferences): + """ + [Section 18.4] + Winner under the Borda count: with k candidates each voter awards k points to + their top choice, k-1 to the next, down to 1 for the last; the candidate with + the highest total score wins. + """ + scores = defaultdict(int) + for ballot in preferences: + for rank, candidate in enumerate(ballot): + scores[candidate] += len(ballot) - rank + return max(scores, key=scores.get) + + +def condorcet_winner(preferences): + """ + [Section 18.4] + The Condorcet winner: the candidate that beats every other candidate in a + pairwise majority comparison. Returns None when no such candidate exists + (Condorcet's paradox), in which case majority preference is cyclic. + """ + candidates = list(preferences[0]) + + def beats(a, b): # a majority of voters rank a above b + votes = sum(ballot.index(a) < ballot.index(b) for ballot in preferences) + return votes > len(preferences) / 2 + + return next((a for a in candidates if all(a == b or beats(a, b) for b in candidates)), None) + + +def vickrey_auction(bids): + """ + [Section 18.4] + Sealed-bid second-price (Vickrey) auction: the highest bidder wins but pays + only the second-highest bid. 'bids' maps each bidder to their bid. Returns the + (winner, price) pair. Because the winner does not pay their own bid, bidding + one's true value is a dominant strategy (the mechanism is truth-revealing). + """ + ranked = sorted(bids.values(), reverse=True) + winner = max(bids, key=bids.get) + price = ranked[1] if len(ranked) > 1 else ranked[0] + return winner, price + + +def contract_net(tasks, agents, bid, select=min): + """ + [Section 18.4.1] + The contract net protocol for task allocation. For each task the manager + broadcasts a task announcement; every agent submits a bid through the callable + bid(agent, task), which returns a numeric bid or None when the agent cannot or + will not perform the task. The manager then awards the task to the agent with + the best bid ('select' = min for costs, max for values). Returns a dict mapping + each task to an (agent, bid) award, or to None if nobody bid. + """ + allocation = {} + for task in tasks: + bids = {agent: bid(agent, task) for agent in agents} + bids = {agent: b for agent, b in bids.items() if b is not None} + allocation[task] = (select(bids, key=bids.get), select(bids.values())) if bids else None + return allocation + + +def alternating_offers_bargaining(discount_a, discount_b): + """ + [Section 18.4.4] + Rubinstein's alternating-offers bargaining over how to split a pie of size 1 + between two impatient agents with discount factors discount_a and discount_b + in [0, 1). In the unique subgame-perfect equilibrium the agent who makes the + first offer (A) keeps a share (1 - discount_b) / (1 - discount_a * discount_b) + and B receives the rest; the more patient an agent is (larger discount factor) + the larger the share it secures. Returns the (share_a, share_b) pair. + """ + share_a = (1 - discount_b) / (1 - discount_a * discount_b) + return share_a, 1 - share_a diff --git a/q-learning-9b/aima/games.py b/q-learning-9b/aima/games.py new file mode 100644 index 0000000..533ebc2 --- /dev/null +++ b/q-learning-9b/aima/games.py @@ -0,0 +1,672 @@ +"""Games or Adversarial Search (Chapter 5)""" + +import copy +import itertools +import random +from collections import namedtuple + +import numpy as np + +from aima.utils import vector_add, MCT_Node, ucb + +GameState = namedtuple('GameState', 'to_move, utility, board, moves') +StochasticGameState = namedtuple('StochasticGameState', 'to_move, utility, board, moves, chance') + + +# ______________________________________________________________________________ +# MinMax Search + + +def minmax_decision(state, game): + """Given a state in a game, calculate the best move by searching + forward all the way to the terminal states. [Figure 5.3]""" + + player = game.to_move(state) + + def max_value(state): + if game.terminal_test(state): + return game.utility(state, player) + v = -np.inf + for a in game.actions(state): + v = max(v, min_value(game.result(state, a))) + return v + + def min_value(state): + if game.terminal_test(state): + return game.utility(state, player) + v = np.inf + for a in game.actions(state): + v = min(v, max_value(game.result(state, a))) + return v + + # Body of minmax_decision: + return max(game.actions(state), key=lambda a: min_value(game.result(state, a))) + + +# ______________________________________________________________________________ + + +def expect_minmax(state, game): + """ + [Figure 5.11] + Return the best move for a player after dice are thrown. The game tree + includes chance nodes along with min and max nodes. + """ + player = game.to_move(state) + + def max_value(state): + v = -np.inf + for a in game.actions(state): + v = max(v, chance_node(state, a)) + return v + + def min_value(state): + v = np.inf + for a in game.actions(state): + v = min(v, chance_node(state, a)) + return v + + def chance_node(state, action): + res_state = game.result(state, action) + if game.terminal_test(res_state): + return game.utility(res_state, player) + sum_chances = 0 + num_chances = len(game.chances(res_state)) + for chance in game.chances(res_state): + res_state = game.outcome(res_state, chance) + util = 0 + if res_state.to_move == player: + util = max_value(res_state) + else: + util = min_value(res_state) + sum_chances += util * game.probability(chance) + return sum_chances / num_chances + + # Body of expect_minmax: + return max(game.actions(state), key=lambda a: chance_node(state, a), default=None) + + +def alpha_beta_search(state, game): + """Search game to determine best action; use alpha-beta pruning. + As in [Figure 5.7], this version searches all the way to the leaves.""" + + player = game.to_move(state) + + # Functions used by alpha_beta + def max_value(state, alpha, beta): + if game.terminal_test(state): + return game.utility(state, player) + v = -np.inf + for a in game.actions(state): + v = max(v, min_value(game.result(state, a), alpha, beta)) + if v >= beta: + return v + alpha = max(alpha, v) + return v + + def min_value(state, alpha, beta): + if game.terminal_test(state): + return game.utility(state, player) + v = np.inf + for a in game.actions(state): + v = min(v, max_value(game.result(state, a), alpha, beta)) + if v <= alpha: + return v + beta = min(beta, v) + return v + + # Body of alpha_beta_search: + best_score = -np.inf + beta = np.inf + best_action = None + for a in game.actions(state): + v = min_value(game.result(state, a), best_score, beta) + if v > best_score: + best_score = v + best_action = a + return best_action + + +def alpha_beta_cutoff_search(state, game, d=4, cutoff_test=None, eval_fn=None): + """Search game to determine best action; use alpha-beta pruning. + This version cuts off search and uses an evaluation function.""" + + player = game.to_move(state) + + # Functions used by alpha_beta + def max_value(state, alpha, beta, depth): + if cutoff_test(state, depth): + return eval_fn(state) + v = -np.inf + for a in game.actions(state): + v = max(v, min_value(game.result(state, a), alpha, beta, depth + 1)) + if v >= beta: + return v + alpha = max(alpha, v) + return v + + def min_value(state, alpha, beta, depth): + if cutoff_test(state, depth): + return eval_fn(state) + v = np.inf + for a in game.actions(state): + v = min(v, max_value(game.result(state, a), alpha, beta, depth + 1)) + if v <= alpha: + return v + beta = min(beta, v) + return v + + # Body of alpha_beta_cutoff_search starts here: + # The default test cuts off at depth d or at a terminal state + cutoff_test = (cutoff_test or (lambda state, depth: depth > d or game.terminal_test(state))) + eval_fn = eval_fn or (lambda state: game.utility(state, player)) + best_score = -np.inf + beta = np.inf + best_action = None + for a in game.actions(state): + v = min_value(game.result(state, a), best_score, beta, 1) + if v > best_score: + best_score = v + best_action = a + return best_action + + +# ______________________________________________________________________________ +# Monte Carlo Tree Search + + +def monte_carlo_tree_search(state, game, N=1000): + """Choose a move by running N iterations of Monte Carlo tree search from the + given state, repeatedly selecting a leaf via UCB, expanding it, simulating a + random playout, and backing the result up; return the most-visited child move.""" + def select(n): + """select a leaf node in the tree""" + if n.children: + return select(max(n.children.keys(), key=ucb)) + else: + return n + + def expand(n): + """expand the leaf node by adding all its children states""" + if not n.children and not game.terminal_test(n.state): + n.children = {MCT_Node(state=game.result(n.state, action), parent=n): action + for action in game.actions(n.state)} + return select(n) + + def simulate(game, state): + """simulate the utility of current state by random picking a step""" + player = game.to_move(state) + while not game.terminal_test(state): + action = random.choice(list(game.actions(state))) + state = game.result(state, action) + v = game.utility(state, player) + return -v + + def backprop(n, utility): + """passing the utility back to all parent nodes""" + if utility > 0: + n.U += utility + # if utility == 0: + # n.U += 0.5 + n.N += 1 + if n.parent: + backprop(n.parent, -utility) + + root = MCT_Node(state=state) + + for _ in range(N): + leaf = select(root) + child = expand(leaf) + result = simulate(game, child.state) + backprop(child, result) + + max_state = max(root.children, key=lambda p: p.N) + + return root.children.get(max_state) + + +# ______________________________________________________________________________ +# Players for Games + + +def query_player(game, state): + """Make a move by querying standard input.""" + print("current state:") + game.display(state) + print("available moves: {}".format(game.actions(state))) + print("") + move = None + if game.actions(state): + while True: + move_string = input('Your move? ') + try: + move = eval(move_string) + except NameError: + move = move_string + if move in game.actions(state): + break + print('illegal move, try again') + else: + print('no legal moves: passing turn to next player') + return move + + +def random_player(game, state): + """A player that chooses a legal move at random.""" + return random.choice(game.actions(state)) if game.actions(state) else None + + +def alpha_beta_player(game, state): + """A player that picks a move using full alpha-beta search.""" + return alpha_beta_search(state, game) + + +def minmax_player(game,state): + """A player that picks a move using full minimax search.""" + return minmax_decision(state,game) + + +def expect_minmax_player(game, state): + """A player for stochastic games that picks a move using expectiminimax search.""" + return expect_minmax(state, game) + + +def mcts_player(game, state): + """A player that picks a move using Monte Carlo tree search.""" + return monte_carlo_tree_search(state, game) + + +# ______________________________________________________________________________ +# Some Sample Games + + +class Game: + """A game is similar to a problem, but it has a utility for each + state and a terminal test instead of a path cost and a goal + test. To create a game, subclass this class and implement actions, + result, utility, and terminal_test. You may override display and + successors or you can inherit their default methods. You will also + need to set the .initial attribute to the initial state; this can + be done in the constructor.""" + + def actions(self, state): + """Return a list of the allowable moves at this point.""" + raise NotImplementedError + + def result(self, state, move): + """Return the state that results from making a move from a state.""" + raise NotImplementedError + + def utility(self, state, player): + """Return the value of this final state to player.""" + raise NotImplementedError + + def terminal_test(self, state): + """Return True if this is a final state for the game.""" + return not self.actions(state) + + def to_move(self, state): + """Return the player whose move it is in this state.""" + return state.to_move + + def display(self, state): + """Print or otherwise display the state.""" + print(state) + + def __repr__(self): + return '<{}>'.format(self.__class__.__name__) + + def play_game(self, *players): + """Play an n-person, move-alternating game.""" + state = self.initial + while True: + for player in players: + move = player(self, state) + state = self.result(state, move) + if self.terminal_test(state): + self.display(state) + return self.utility(state, self.to_move(self.initial)) + + +class StochasticGame(Game): + """A stochastic game includes uncertain events which influence + the moves of players at each state. To create a stochastic game, subclass + this class and implement chances and outcome along with the other + unimplemented game class methods.""" + + def chances(self, state): + """Return a list of all possible uncertain events at a state.""" + raise NotImplementedError + + def outcome(self, state, chance): + """Return the state which is the outcome of a chance trial.""" + raise NotImplementedError + + def probability(self, chance): + """Return the probability of occurrence of a chance.""" + raise NotImplementedError + + def play_game(self, *players): + """Play an n-person, move-alternating stochastic game.""" + state = self.initial + while True: + for player in players: + chance = random.choice(self.chances(state)) + state = self.outcome(state, chance) + move = player(self, state) + state = self.result(state, move) + if self.terminal_test(state): + self.display(state) + return self.utility(state, self.to_move(self.initial)) + + +class Fig52Game(Game): + """The game represented in [Figure 5.2]. Serves as a simple test case.""" + + succs = dict(A=dict(a1='B', a2='C', a3='D'), + B=dict(b1='B1', b2='B2', b3='B3'), + C=dict(c1='C1', c2='C2', c3='C3'), + D=dict(d1='D1', d2='D2', d3='D3')) + utils = dict(B1=3, B2=12, B3=8, C1=2, C2=4, C3=6, D1=14, D2=5, D3=2) + initial = 'A' + + def actions(self, state): + """Return the moves available from the given node of the game tree.""" + return list(self.succs.get(state, {}).keys()) + + def result(self, state, move): + """Return the successor node reached by taking the given move.""" + return self.succs[state][move] + + def utility(self, state, player): + """Return the leaf value, negated for the MIN player.""" + if player == 'MAX': + return self.utils[state] + else: + return -self.utils[state] + + def terminal_test(self, state): + """Return True for leaf nodes, i.e. anything other than A, B, C, D.""" + return state not in ('A', 'B', 'C', 'D') + + def to_move(self, state): + """Return 'MIN' for the second-ply nodes B, C, D, otherwise 'MAX'.""" + return 'MIN' if state in 'BCD' else 'MAX' + + +class Fig52Extended(Game): + """Similar to Fig52Game but bigger. Useful for visualisation""" + + succs = {i: dict(l=i * 3 + 1, m=i * 3 + 2, r=i * 3 + 3) for i in range(13)} + utils = dict() + + def actions(self, state): + """Return the moves, sorted, available from the given node.""" + return sorted(list(self.succs.get(state, {}).keys())) + + def result(self, state, move): + """Return the successor node reached by taking the given move.""" + return self.succs[state][move] + + def utility(self, state, player): + """Return the leaf value, negated for the MIN player.""" + if player == 'MAX': + return self.utils[state] + else: + return -self.utils[state] + + def terminal_test(self, state): + """Return True once the state falls outside the 0-12 node range.""" + return state not in range(13) + + def to_move(self, state): + """Return 'MIN' for the second-ply nodes 1, 2, 3, otherwise 'MAX'.""" + return 'MIN' if state in {1, 2, 3} else 'MAX' + + +class TicTacToe(Game): + """Play TicTacToe on an h x v board, with Max (first player) playing 'X'. + A state has the player to move, a cached utility, a list of moves in + the form of a list of (x, y) positions, and a board, in the form of + a dict of {(x, y): Player} entries, where Player is 'X' or 'O'.""" + + def __init__(self, h=3, v=3, k=3): + self.h = h + self.v = v + self.k = k + moves = [(x, y) for x in range(1, h + 1) + for y in range(1, v + 1)] + self.initial = GameState(to_move='X', utility=0, board={}, moves=moves) + + def actions(self, state): + """Legal moves are any square not yet taken.""" + return state.moves + + def result(self, state, move): + """Place the current player's mark at move and return the new state; + an illegal move leaves the state unchanged.""" + if move not in state.moves: + return state # Illegal move has no effect + board = state.board.copy() + board[move] = state.to_move + moves = list(state.moves) + moves.remove(move) + return GameState(to_move=('O' if state.to_move == 'X' else 'X'), + utility=self.compute_utility(board, move, state.to_move), + board=board, moves=moves) + + def utility(self, state, player): + """Return the value to player; 1 for win, -1 for loss, 0 otherwise.""" + return state.utility if player == 'X' else -state.utility + + def terminal_test(self, state): + """A state is terminal if it is won or there are no empty squares.""" + return state.utility != 0 or len(state.moves) == 0 + + def display(self, state): + """Print the board as a grid, with empty squares shown as dots.""" + board = state.board + for x in range(1, self.h + 1): + for y in range(1, self.v + 1): + print(board.get((x, y), '.'), end=' ') + print() + + def compute_utility(self, board, move, player): + """If 'X' wins with this move, return 1; if 'O' wins return -1; else return 0.""" + if (self.k_in_row(board, move, player, (0, 1)) or + self.k_in_row(board, move, player, (1, 0)) or + self.k_in_row(board, move, player, (1, -1)) or + self.k_in_row(board, move, player, (1, 1))): + return +1 if player == 'X' else -1 + else: + return 0 + + def k_in_row(self, board, move, player, delta_x_y): + """Return true if there is a line through move on board for player.""" + (delta_x, delta_y) = delta_x_y + x, y = move + n = 0 # n is number of moves in row + while board.get((x, y)) == player: + n += 1 + x, y = x + delta_x, y + delta_y + x, y = move + while board.get((x, y)) == player: + n += 1 + x, y = x - delta_x, y - delta_y + n -= 1 # Because we counted move itself twice + return n >= self.k + + +class ConnectFour(TicTacToe): + """A TicTacToe-like game in which you can only make a move on the bottom + row, or in a square directly above an occupied square. Traditionally + played on a 7x6 board and requiring 4 in a row.""" + + def __init__(self, h=7, v=6, k=4): + TicTacToe.__init__(self, h, v, k) + + def actions(self, state): + """Return legal moves: bottom-row squares or squares directly above an occupied one.""" + return [(x, y) for (x, y) in state.moves + if x == self.h or (x + 1 , y ) in state.board] + +class Gomoku(TicTacToe): + """Also known as Five in a row.""" + + def __init__(self, h=15, v=16, k=5): + TicTacToe.__init__(self, h, v, k) + + +class Backgammon(StochasticGame): + """A two player game where the goal of each player is to move all the + checkers off the board. The moves for each state are determined by + rolling a pair of dice.""" + + def __init__(self): + """Initial state of the game""" + point = {'W': 0, 'B': 0} + board = [point.copy() for index in range(24)] + board[0]['B'] = board[23]['W'] = 2 + board[5]['W'] = board[18]['B'] = 5 + board[7]['W'] = board[16]['B'] = 3 + board[11]['B'] = board[12]['W'] = 5 + self.allow_bear_off = {'W': False, 'B': False} + self.direction = {'W': -1, 'B': 1} + self.initial = StochasticGameState(to_move='W', + utility=0, + board=board, + moves=self.get_all_moves(board, 'W'), chance=None) + + def actions(self, state): + """Return a list of legal moves for a state.""" + player = state.to_move + moves = state.moves + if len(moves) == 1 and len(moves[0]) == 1: + return moves + legal_moves = [] + for move in moves: + board = copy.deepcopy(state.board) + if self.is_legal_move(board, move, state.chance, player): + legal_moves.append(move) + return legal_moves + + def result(self, state, move): + """Apply the player's checker move(s) for the current dice roll and return + the resulting state, with the turn passed to the opponent.""" + board = copy.deepcopy(state.board) + player = state.to_move + self.move_checker(board, move[0], state.chance[0], player) + if len(move) == 2: + self.move_checker(board, move[1], state.chance[1], player) + to_move = ('W' if player == 'B' else 'B') + return StochasticGameState(to_move=to_move, + utility=self.compute_utility(board, move, player), + board=board, + moves=self.get_all_moves(board, to_move), chance=None) + + def utility(self, state, player): + """Return the value to player; 1 for win, -1 for loss, 0 otherwise.""" + return state.utility if player == 'W' else -state.utility + + def terminal_test(self, state): + """A state is terminal if one player wins.""" + return state.utility != 0 + + def get_all_moves(self, board, player): + """All possible moves for a player i.e. all possible ways of + choosing two checkers of a player from the board for a move + at a given state.""" + all_points = board + taken_points = [index for index, point in enumerate(all_points) + if point[player] > 0] + if self.checkers_at_home(board, player) == 1: + return [(taken_points[0],)] + moves = list(itertools.permutations(taken_points, 2)) + moves = moves + [(index, index) for index, point in enumerate(all_points) + if point[player] >= 2] + return moves + + def display(self, state): + """Display state of the game.""" + board = state.board + player = state.to_move + print("current state : ") + for index, point in enumerate(board): + print("point : ", index, " W : ", point['W'], " B : ", point['B']) + print("to play : ", player) + + def compute_utility(self, board, move, player): + """If 'W' wins with this move, return 1; if 'B' wins return -1; else return 0.""" + util = {'W': 1, 'B': -1} + for idx in range(0, 24): + if board[idx][player] > 0: + return 0 + return util[player] + + def checkers_at_home(self, board, player): + """Return the no. of checkers at home for a player.""" + sum_range = range(0, 7) if player == 'W' else range(17, 24) + count = 0 + for idx in sum_range: + count = count + board[idx][player] + return count + + def is_legal_move(self, board, start, steps, player): + """Move is a tuple which contains starting points of checkers to be + moved during a player's turn. An on-board move is legal if both the destinations + are open. A bear-off move is the one where a checker is moved off-board. + It is legal only after a player has moved all his checkers to his home.""" + dest1, dest2 = vector_add(start, steps) + dest_range = range(0, 24) + move1_legal = move2_legal = False + if dest1 in dest_range: + if self.is_point_open(player, board[dest1]): + self.move_checker(board, start[0], steps[0], player) + move1_legal = True + else: + if self.allow_bear_off[player]: + self.move_checker(board, start[0], steps[0], player) + move1_legal = True + if not move1_legal: + return False + if dest2 in dest_range: + if self.is_point_open(player, board[dest2]): + move2_legal = True + else: + if self.allow_bear_off[player]: + move2_legal = True + return move1_legal and move2_legal + + def move_checker(self, board, start, steps, player): + """Move a checker from starting point by a given number of steps""" + dest = start + steps + dest_range = range(0, 24) + board[start][player] -= 1 + if dest in dest_range: + board[dest][player] += 1 + if self.checkers_at_home(board, player) == 15: + self.allow_bear_off[player] = True + + def is_point_open(self, player, point): + """A point is open for a player if the no. of opponent's + checkers already present on it is 0 or 1. A player can + move a checker to a point only if it is open.""" + opponent = 'B' if player == 'W' else 'W' + return point[opponent] <= 1 + + def chances(self, state): + """Return a list of all possible dice rolls at a state.""" + dice_rolls = list(itertools.combinations_with_replacement([1, 2, 3, 4, 5, 6], 2)) + return dice_rolls + + def outcome(self, state, chance): + """Return the state which is the outcome of a dice roll.""" + dice = tuple(map((self.direction[state.to_move]).__mul__, chance)) + return StochasticGameState(to_move=state.to_move, + utility=state.utility, + board=state.board, + moves=state.moves, chance=dice) + + def probability(self, chance): + """Return the probability of occurrence of a dice roll.""" + return 1 / 36 if chance[0] == chance[1] else 1 / 18 diff --git a/q-learning-9b/aima/ipyviews.py b/q-learning-9b/aima/ipyviews.py new file mode 100644 index 0000000..3976cc7 --- /dev/null +++ b/q-learning-9b/aima/ipyviews.py @@ -0,0 +1,175 @@ +from IPython.display import HTML, display, clear_output +from collections import defaultdict +from aima.agents import PolygonObstacle +import os +import time +import json +import copy +import __main__ + +# repo root (the directory containing the `aima` package), so the bundled js/ +# files load regardless of the current working directory +_AIMA_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# ______________________________________________________________________________ +# Continuous environment + + +_CONTINUOUS_WORLD_HTML = ''' +
+ +
+ + +''' # noqa + +with open(os.path.join(_AIMA_ROOT, 'js/continuousworld.js'), 'r') as js_file: + _JS_CONTINUOUS_WORLD = js_file.read() + + +class ContinuousWorldView: + """ View for continuousworld Implementation in agents.py """ + + def __init__(self, world, fill="#AAA"): + self.time = time.time() + self.world = world + self.width = world.width + self.height = world.height + + def object_name(self): + """Return the variable name this view is bound to in the main namespace, + matched by its creation timestamp, so the JavaScript canvas can address it.""" + globals_in_main = {x: getattr(__main__, x) for x in dir(__main__)} + for x in globals_in_main: + if isinstance(globals_in_main[x], type(self)): + if globals_in_main[x].time == self.time: + return x + + def handle_add_obstacle(self, vertices): + """ Vertices must be a nestedtuple. This method + is called from kernel.execute on completion of + a polygon. """ + self.world.add_obstacle(vertices) + self.show() + + def handle_remove_obstacle(self): + """Hook for removing an obstacle from the world; not yet implemented.""" + raise NotImplementedError + + def get_polygon_obstacles_coordinates(self): + """Return the vertex coordinates of every polygon obstacle in the world.""" + obstacle_coordiantes = [] + for thing in self.world.things: + if isinstance(thing, PolygonObstacle): + obstacle_coordiantes.append(thing.coordinates) + return obstacle_coordiantes + + def show(self): + """Render the continuous world (its dimensions and obstacles) as an HTML + canvas and display it in the notebook output.""" + clear_output() + total_html = _CONTINUOUS_WORLD_HTML.format(self.width, self.height, self.object_name(), + str(self.get_polygon_obstacles_coordinates()), + _JS_CONTINUOUS_WORLD) + display(HTML(total_html)) + + +# ______________________________________________________________________________ +# Grid environment + +_GRID_WORLD_HTML = ''' +
+ +
+ +
+
+ +''' + +with open(os.path.join(_AIMA_ROOT, 'js/gridworld.js'), 'r') as js_file: + _JS_GRID_WORLD = js_file.read() + + +class GridWorldView: + """ View for grid world. Uses XYEnviornment in agents.py as model. + world: an instance of XYEnviornment. + block_size: size of individual blocks in pixels. + default_fill: color of blocks. A hex value or name should be passed. + """ + + def __init__(self, world, block_size=30, default_fill="white"): + self.time = time.time() + self.world = world + self.labels = defaultdict(str) # locations as keys + self.representation = {"default": {"type": "color", "source": default_fill}} + self.block_size = block_size + + def object_name(self): + """Return the variable name this view is bound to in the main namespace, + matched by its creation timestamp, so the JavaScript canvas can address it.""" + globals_in_main = {x: getattr(__main__, x) for x in dir(__main__)} + for x in globals_in_main: + if isinstance(globals_in_main[x], type(self)): + if globals_in_main[x].time == self.time: + return x + + def set_label(self, coordinates, label): + """ Add labels to a particular block of grid. + coordinates: a tuple of (row, column). + rows and columns are 0 indexed. + """ + self.labels[coordinates] = label + + def set_representation(self, thing, repr_type, source): + """ Set the representation of different things in the + environment. + thing: a thing object. + repr_type : type of representation can be either "color" or "img" + source: Hex value in case of color. Image path in case of image. + """ + thing_class_name = thing.__class__.__name__ + if repr_type not in ("img", "color"): + raise ValueError('Invalid repr_type passed. Possible types are img/color') + self.representation[thing_class_name] = {"type": repr_type, "source": source} + + def handle_click(self, coordinates): + """ This method needs to be overridden. Make sure to include a + self.show() call at the end. """ + self.show() + + def map_to_render(self): + """Build the grid's render model and return it as a JSON string: a 2D array of + cells annotated with each thing's class name and any per-location tooltip + label.""" + default_representation = {"val": "default", "tooltip": ""} + world_map = [[copy.deepcopy(default_representation) for _ in range(self.world.width)] + for _ in range(self.world.height)] + + for thing in self.world.things: + row, column = thing.location + thing_class_name = thing.__class__.__name__ + if thing_class_name not in self.representation: + raise KeyError('Representation not found for {}'.format(thing_class_name)) + world_map[row][column]["val"] = thing.__class__.__name__ + + for location, label in self.labels.items(): + row, column = location + world_map[row][column]["tooltip"] = label + + return json.dumps(world_map) + + def show(self): + """Render the grid world (its cells, sizes, and representations) as an HTML + canvas and display it in the notebook output.""" + clear_output() + total_html = _GRID_WORLD_HTML.format( + self.object_name(), self.map_to_render(), + self.block_size, json.dumps(self.representation), _JS_GRID_WORLD) + display(HTML(total_html)) diff --git a/q-learning-9b/aima/knowledge.py b/q-learning-9b/aima/knowledge.py new file mode 100644 index 0000000..66d9adc --- /dev/null +++ b/q-learning-9b/aima/knowledge.py @@ -0,0 +1,431 @@ +"""Knowledge in learning (Chapter 19)""" + +from collections import defaultdict +from functools import partial +from itertools import combinations, product +from random import shuffle + +import numpy as np + +from aima.logic import (FolKB, constant_symbols, predicate_symbols, standardize_variables, + variables, is_definite_clause, subst, expr, Expr) +from aima.utils import power_set + + +def current_best_learning(examples, h, examples_so_far=None): + """ + [Figure 19.2] + The hypothesis is a list of dictionaries, with each dictionary representing + a disjunction. + """ + if examples_so_far is None: + examples_so_far = [] + if not examples: + return h + + e = examples[0] + if is_consistent(e, h): + return current_best_learning(examples[1:], h, examples_so_far + [e]) + elif false_positive(e, h): + for h2 in specializations(examples_so_far + [e], h): + h3 = current_best_learning(examples[1:], h2, examples_so_far + [e]) + if h3 != 'FAIL': + return h3 + elif false_negative(e, h): + for h2 in generalizations(examples_so_far + [e], h): + h3 = current_best_learning(examples[1:], h2, examples_so_far + [e]) + if h3 != 'FAIL': + return h3 + + return 'FAIL' + + +def specializations(examples_so_far, h): + """Specialize the hypothesis by adding AND operations to the disjunctions""" + hypotheses = [] + + for i, disj in enumerate(h): + for e in examples_so_far: + for k, v in e.items(): + if k in disj or k == 'GOAL': + continue + + h2 = h[i].copy() + h2[k] = '!' + v + h3 = h.copy() + h3[i] = h2 + if check_all_consistency(examples_so_far, h3): + hypotheses.append(h3) + + shuffle(hypotheses) + return hypotheses + + +def generalizations(examples_so_far, h): + """Generalize the hypothesis. First delete operations + (including disjunctions) from the hypothesis. Then, add OR operations.""" + hypotheses = [] + + # Delete disjunctions + disj_powerset = power_set(range(len(h))) + for disjs in disj_powerset: + h2 = h.copy() + for d in reversed(list(disjs)): + del h2[d] + + if check_all_consistency(examples_so_far, h2): + hypotheses += h2 + + # Delete AND operations in disjunctions + for i, disj in enumerate(h): + a_powerset = power_set(disj.keys()) + for attrs in a_powerset: + h2 = h[i].copy() + for a in attrs: + del h2[a] + + if check_all_consistency(examples_so_far, [h2]): + h3 = h.copy() + h3[i] = h2.copy() + hypotheses += h3 + + # Add OR operations + if hypotheses == [] or hypotheses == [{}]: + hypotheses = add_or(examples_so_far, h) + else: + hypotheses.extend(add_or(examples_so_far, h)) + + shuffle(hypotheses) + return hypotheses + + +def add_or(examples_so_far, h): + """Add an OR operation to the hypothesis. The AND operations in the disjunction + are generated by the last example (which is the problematic one).""" + ors = [] + e = examples_so_far[-1] + + attrs = {k: v for k, v in e.items() if k != 'GOAL'} + a_powerset = power_set(attrs.keys()) + + for c in a_powerset: + h2 = {} + for k in c: + h2[k] = attrs[k] + + if check_negative_consistency(examples_so_far, h2): + h3 = h.copy() + h3.append(h2) + ors.append(h3) + + return ors + + +# ______________________________________________________________________________ + + +def version_space_learning(examples): + """ + [Figure 19.3] + The version space is a list of hypotheses, which in turn are a list + of dictionaries/disjunctions. + """ + V = all_hypotheses(examples) + for e in examples: + if V: + V = version_space_update(V, e) + + return V + + +def version_space_update(V, e): + """Return the version space ``V`` restricted to the hypotheses that remain + consistent with example ``e``.""" + return [h for h in V if is_consistent(e, h)] + + +def all_hypotheses(examples): + """Build a list of all the possible hypotheses""" + values = values_table(examples) + h_powerset = power_set(values.keys()) + hypotheses = [] + for s in h_powerset: + hypotheses.extend(build_attr_combinations(s, values)) + + hypotheses.extend(build_h_combinations(hypotheses)) + + return hypotheses + + +def values_table(examples): + """Build a table with all the possible values for each attribute. + Returns a dictionary with keys the attribute names and values a list + with the possible values for the corresponding attribute.""" + values = defaultdict(lambda: []) + for e in examples: + for k, v in e.items(): + if k == 'GOAL': + continue + + mod = '!' + if e['GOAL']: + mod = '' + + if mod + v not in values[k]: + values[k].append(mod + v) + + values = dict(values) + return values + + +def build_attr_combinations(s, values): + """Given a set of attributes, builds all the combinations of values. + If the set holds more than one attribute, recursively builds the + combinations.""" + if len(s) == 1: + # s holds just one attribute, return its list of values + k = values[s[0]] + h = [[{s[0]: v}] for v in values[s[0]]] + return h + + h = [] + for i, a in enumerate(s): + rest = build_attr_combinations(s[i + 1:], values) + for v in values[a]: + o = {a: v} + for r in rest: + t = o.copy() + for d in r: + t.update(d) + h.append([t]) + + return h + + +def build_h_combinations(hypotheses): + """Given a set of hypotheses, builds and returns all the combinations of the + hypotheses.""" + h = [] + h_powerset = power_set(range(len(hypotheses))) + + for s in h_powerset: + t = [] + for i in s: + t.extend(hypotheses[i]) + h.append(t) + + return h + + +# ______________________________________________________________________________ + + +def minimal_consistent_det(E, A): + """Return a minimal set of attributes which give consistent determination""" + n = len(A) + + for i in range(n + 1): + for A_i in combinations(A, i): + if consistent_det(A_i, E): + return set(A_i) + + +def consistent_det(A, E): + """Check if the attributes(A) is consistent with the examples(E)""" + H = {} + + for e in E: + attr_values = tuple(e[attr] for attr in A) + if attr_values in H and H[attr_values] != e['GOAL']: + return False + H[attr_values] = e['GOAL'] + + return True + + +# ______________________________________________________________________________ + + +class FOILContainer(FolKB): + """Hold the kb and other necessary elements required by FOIL.""" + + def __init__(self, clauses=None): + self.const_syms = set() + self.pred_syms = set() + super().__init__(clauses) + + def tell(self, sentence): + """Add a definite clause to the knowledge base, recording its constant and + predicate symbols; raise an exception if it is not a definite clause.""" + if is_definite_clause(sentence): + self.clauses.append(sentence) + self.const_syms.update(constant_symbols(sentence)) + self.pred_syms.update(predicate_symbols(sentence)) + else: + raise Exception('Not a definite clause: {}'.format(sentence)) + + def foil(self, examples, target): + """Learn a list of first-order horn clauses + 'examples' is a tuple: (positive_examples, negative_examples). + positive_examples and negative_examples are both lists which contain substitutions.""" + clauses = [] + + pos_examples = examples[0] + neg_examples = examples[1] + + while pos_examples: + clause, extended_pos_examples = self.new_clause((pos_examples, neg_examples), target) + # remove positive examples covered by clause + pos_examples = self.update_examples(target, pos_examples, extended_pos_examples) + clauses.append(clause) + + return clauses + + def new_clause(self, examples, target): + """Find a horn clause which satisfies part of the positive + examples but none of the negative examples. + The horn clause is specified as [consequent, list of antecedents] + Return value is the tuple (horn_clause, extended_positive_examples).""" + clause = [target, []] + extended_examples = examples + while extended_examples[1]: + l = self.choose_literal(self.new_literals(clause), extended_examples) + clause[1].append(l) + extended_examples = [sum([list(self.extend_example(example, l)) for example in + extended_examples[i]], []) for i in range(2)] + + return clause, extended_examples[0] + + def extend_example(self, example, literal): + """Generate extended examples which satisfy the literal.""" + # find all substitutions that satisfy literal + for s in self.ask_generator(subst(example, literal)): + s.update(example) + yield s + + def new_literals(self, clause): + """Generate new literals based on known predicate symbols. + Generated literal must share at least one variable with clause""" + share_vars = variables(clause[0]) + for l in clause[1]: + share_vars.update(variables(l)) + for pred, arity in self.pred_syms: + new_vars = {standardize_variables(expr('x')) for _ in range(arity - 1)} + for args in product(share_vars.union(new_vars), repeat=arity): + if any(var in share_vars for var in args): + # make sure we don't return an existing rule + if not Expr(pred, args) in clause[1]: + yield Expr(pred, *[var for var in args]) + + def choose_literal(self, literals, examples): + """Choose the best literal based on the information gain.""" + return max(literals, key=partial(self.gain, examples=examples)) + + def gain(self, l, examples): + """ + Find the utility of each literal when added to the body of the clause. + Utility function is:: + + gain(R, l) = T * (log_2 (post_pos / (post_pos + post_neg)) - log_2 (pre_pos / (pre_pos + pre_neg))) + + where:: + + pre_pos = number of positive bindings of rule R (=current set of rules) + pre_neg = number of negative bindings of rule R + post_pos = number of positive bindings of rule R' (= R U {l} ) + post_neg = number of negative bindings of rule R' + T = number of positive bindings of rule R that are still covered + after adding literal l + + """ + pre_pos = len(examples[0]) + pre_neg = len(examples[1]) + post_pos = sum([list(self.extend_example(example, l)) for example in examples[0]], []) + post_neg = sum([list(self.extend_example(example, l)) for example in examples[1]], []) + if pre_pos + pre_neg == 0 or len(post_pos) + len(post_neg) == 0: + return -1 + # number of positive example that are represented in extended_examples + T = 0 + for example in examples[0]: + represents = lambda d: all(d[x] == example[x] for x in example) + if any(represents(l_) for l_ in post_pos): + T += 1 + value = T * (np.log2(len(post_pos) / (len(post_pos) + len(post_neg)) + 1e-12) - + np.log2(pre_pos / (pre_pos + pre_neg))) + return value + + def update_examples(self, target, examples, extended_examples): + """Add to the kb those examples what are represented in extended_examples + List of omitted examples is returned.""" + uncovered = [] + for example in examples: + represents = lambda d: all(d[x] == example[x] for x in example) + if any(represents(l) for l in extended_examples): + self.tell(subst(example, target)) + else: + uncovered.append(example) + + return uncovered + + +# ______________________________________________________________________________ + + +def check_all_consistency(examples, h): + """Check for the consistency of all examples under h.""" + for e in examples: + if not is_consistent(e, h): + return False + + return True + + +def check_negative_consistency(examples, h): + """Check if the negative examples are consistent under h.""" + for e in examples: + if e['GOAL']: + continue + + if not is_consistent(e, [h]): + return False + + return True + + +def disjunction_value(e, d): + """The value of example e under disjunction d.""" + for k, v in d.items(): + if v[0] == '!': + # v is a NOT expression + # e[k], thus, should not be equal to v + if e[k] == v[1:]: + return False + elif e[k] != v: + return False + + return True + + +def guess_value(e, h): + """Guess value of example e under hypothesis h.""" + for d in h: + if disjunction_value(e, d): + return True + + return False + + +def is_consistent(e, h): + """Return True iff hypothesis ``h`` predicts the same value as example ``e``'s + actual GOAL label.""" + return e['GOAL'] == guess_value(e, h) + + +def false_positive(e, h): + """Return True iff ``h`` predicts True for ``e`` while its actual GOAL is False.""" + return guess_value(e, h) and not e['GOAL'] + + +def false_negative(e, h): + """Return True iff ``h`` predicts False for ``e`` while its actual GOAL is True.""" + return e['GOAL'] and not guess_value(e, h) diff --git a/q-learning-9b/aima/learning.py b/q-learning-9b/aima/learning.py new file mode 100644 index 0000000..a4efc89 --- /dev/null +++ b/q-learning-9b/aima/learning.py @@ -0,0 +1,1437 @@ +"""Learning from examples (Chapters 18)""" + +import copy +import itertools +from collections import defaultdict +from statistics import stdev + +from aima.probabilistic_learning import NaiveBayesLearner +from aima.utils import * + + +class DataSet: + """ + A data set for a machine learning problem. It has the following fields:: + + d.examples A list of examples. Each one is a list of attribute values. + d.attrs A list of integers to index into an example, so example[attr] + gives a value. Normally the same as range(len(d.examples[0])). + d.attr_names Optional list of mnemonic names for corresponding attrs. + d.target The attribute that a learning algorithm will try to predict. + By default the final attribute. + d.inputs The list of attrs without the target. + d.values A list of lists: each sublist is the set of possible + values for the corresponding attribute. If initially None, + it is computed from the known examples by self.set_problem. + If not None, an erroneous value raises ValueError. + d.distance A function from a pair of examples to a non-negative number. + Should be symmetric, etc. Defaults to mean_boolean_error + since that can handle any field types. + d.name Name of the data set (for output display only). + d.source URL or other source where the data came from. + d.exclude A list of attribute indexes to exclude from d.inputs. Elements + of this list can either be integers (attrs) or attr_names. + + Normally, you call the constructor and you're done; then you just + access fields like d.examples and d.target and d.inputs. + """ + + def __init__(self, examples=None, attrs=None, attr_names=None, target=-1, inputs=None, + values=None, distance=mean_boolean_error, name='', source='', exclude=()): + """ + Accepts any of DataSet's fields. Examples can also be a + string or file from which to parse examples using parse_csv. + Optional parameter: exclude, as documented in .set_problem(). + >>> DataSet(examples='1, 2, 3') + + """ + self.name = name + self.source = source + self.values = values + self.distance = distance + self.got_values_flag = bool(values) + + # initialize .examples from string or list or data directory + if isinstance(examples, str): + self.examples = parse_csv(examples) + elif examples is None: + self.examples = parse_csv(open_data(name + '.csv').read()) + else: + self.examples = examples + + # attrs are the indices of examples, unless otherwise stated. + if self.examples is not None and attrs is None: + attrs = list(range(len(self.examples[0]))) + + self.attrs = attrs + + # initialize .attr_names from string, list, or by default + if isinstance(attr_names, str): + self.attr_names = attr_names.split() + else: + self.attr_names = attr_names or attrs + self.set_problem(target, inputs=inputs, exclude=exclude) + + def set_problem(self, target, inputs=None, exclude=()): + """ + Set (or change) the target and/or inputs. + This way, one DataSet can be used multiple ways. inputs, if specified, + is a list of attributes, or specify exclude as a list of attributes + to not use in inputs. Attributes can be -n .. n, or an attr_name. + Also computes the list of possible values, if that wasn't done yet. + """ + self.target = self.attr_num(target) + exclude = list(map(self.attr_num, exclude)) + if inputs: + self.inputs = remove_all(self.target, inputs) + else: + self.inputs = [a for a in self.attrs if a != self.target and a not in exclude] + if not self.values: + self.update_values() + self.check_me() + + def check_me(self): + """Check that my fields make sense.""" + assert len(self.attr_names) == len(self.attrs) + assert self.target in self.attrs + assert self.target not in self.inputs + assert set(self.inputs).issubset(set(self.attrs)) + if self.got_values_flag: + # only check if values are provided while initializing DataSet + list(map(self.check_example, self.examples)) + + def add_example(self, example): + """Add an example to the list of examples, checking it first.""" + self.check_example(example) + self.examples.append(example) + + def check_example(self, example): + """Raise ValueError if example has any invalid values.""" + if self.values: + for a in self.attrs: + if example[a] not in self.values[a]: + raise ValueError('Bad value {} for attribute {} in {}' + .format(example[a], self.attr_names[a], example)) + + def attr_num(self, attr): + """Returns the number used for attr, which can be a name, or -n .. n-1.""" + if isinstance(attr, str): + return self.attr_names.index(attr) + elif attr < 0: + return len(self.attrs) + attr + else: + return attr + + def update_values(self): + """Recompute ``self.values`` (the list of distinct values per attribute) from the examples.""" + self.values = list(map(unique, zip(*self.examples))) + + def sanitize(self, example): + """Return a copy of example, with non-input attributes replaced by None.""" + return [attr_i if i in self.inputs else None for i, attr_i in enumerate(example)][:-1] + + def classes_to_numbers(self, classes=None): + """Converts class names to numbers.""" + if not classes: + # if classes were not given, extract them from values + classes = sorted(self.values[self.target]) + for item in self.examples: + item[self.target] = classes.index(item[self.target]) + + def remove_examples(self, value=''): + """Remove examples that contain given value.""" + self.examples = [x for x in self.examples if value not in x] + self.update_values() + + def split_values_by_classes(self): + """Split values into buckets according to their class.""" + buckets = defaultdict(lambda: []) + target_names = self.values[self.target] + + for v in self.examples: + item = [a for a in v if a not in target_names] # remove target from item + buckets[v[self.target]].append(item) # add item to bucket of its class + + return buckets + + def find_means_and_deviations(self): + """ + Finds the means and standard deviations of self.dataset:: + + means : a dictionary for each class/target. Holds a list of the means + of the features for the class. + deviations: a dictionary for each class/target. Holds a list of the sample + standard deviations of the features for the class. + """ + target_names = self.values[self.target] + feature_numbers = len(self.inputs) + + item_buckets = self.split_values_by_classes() + + means = defaultdict(lambda: [0] * feature_numbers) + deviations = defaultdict(lambda: [0] * feature_numbers) + + for t in target_names: + # find all the item feature values for item in class t + features = [[] for _ in range(feature_numbers)] + for item in item_buckets[t]: + for i in range(feature_numbers): + features[i].append(item[i]) + + # calculate means and deviations fo the class + for i in range(feature_numbers): + means[t][i] = mean(features[i]) + deviations[t][i] = stdev(features[i]) + + return means, deviations + + def __repr__(self): + return ''.format(self.name, len(self.examples), len(self.attrs)) + + +def parse_csv(input, delim=','): + r""" + Input is a string consisting of lines, each line has comma-delimited + fields. Convert this into a list of lists. Blank lines are skipped. + Fields that look like numbers are converted to numbers. + The delim defaults to ',' but '\t' and None are also reasonable values. + >>> parse_csv('1, 2, 3 \n 0, 2, na') + [[1, 2, 3], [0, 2, 'na']] + """ + lines = [line for line in input.splitlines() if line.strip()] + return [list(map(num_or_str, line.split(delim))) for line in lines] + + +def err_ratio(predict, dataset, examples=None): + """ + Return the proportion of the examples that are NOT correctly predicted. + verbose - 0: No output; 1: Output wrong; 2 (or greater): Output correct + Accepts either a callable predictor or a learner object with a .predict method. + """ + if hasattr(predict, 'predict'): + predict = predict.predict + examples = examples or dataset.examples + if len(examples) == 0: + return 0.0 + right = 0 + for example in examples: + desired = example[dataset.target] + output = predict(dataset.sanitize(example)) + if output == desired: + right += 1 + return 1 - (right / len(examples)) + + +def grade_learner(predict, tests): + """ + Grades the given learner based on how many tests it passes. + tests is a list with each element in the form: (values, output). + Accepts either a callable predictor or a learner object with a .predict method. + """ + if hasattr(predict, 'predict'): + predict = predict.predict + return mean(int(predict(X) == y) for X, y in tests) + + +def train_test_split(dataset, start=None, end=None, test_split=None): + """ + If you are giving 'start' and 'end' as parameters, + then it will return the testing set from index 'start' to 'end' + and the rest for training. + If you give 'test_split' as a parameter then it will return + test_split * 100% as the testing set and the rest as + training set. + """ + examples = dataset.examples + if test_split is None: + train = examples[:start] + examples[end:] + val = examples[start:end] + else: + total_size = len(examples) + val_size = int(total_size * test_split) + train_size = total_size - val_size + train = examples[:train_size] + val = examples[train_size:total_size] + + return train, val + + +def cross_validation_wrapper(learner, dataset, k=10, trials=1): + """ + [Figure 18.8] + Return the optimal value of size having minimum error on validation set. + errT: a training error array, indexed by size + errV: a validation error array, indexed by size + """ + errs = [] + size = 1 + while True: + errT, errV = cross_validation(learner, dataset, size, k, trials) + # check for convergence provided err_val is not empty + if errT and not np.isclose(errT[-1], errT, rtol=1e-6): + best_size = 0 + min_val = np.inf + i = 0 + while i < size: + if errs[i] < min_val: + min_val = errs[i] + best_size = i + i += 1 + return learner(dataset, best_size) + errs.append(errV) + size += 1 + + +def cross_validation(learner, dataset, size=None, k=10, trials=1): + """ + Do k-fold cross_validate and return their mean. + That is, keep out 1/k of the examples for testing on each of k runs. + Shuffle the examples first; if trials > 1, average over several shuffles. + Returns Training error, Validation error + """ + k = k or len(dataset.examples) + if trials > 1: + trial_errT = 0 + trial_errV = 0 + for t in range(trials): + errT, errV = cross_validation(learner, dataset, size, k, trials) + trial_errT += errT + trial_errV += errV + return trial_errT / trials, trial_errV / trials + else: + fold_errT = 0 + fold_errV = 0 + n = len(dataset.examples) + examples = dataset.examples + random.shuffle(dataset.examples) + for fold in range(k): + train_data, val_data = train_test_split(dataset, fold * (n // k), (fold + 1) * (n // k)) + dataset.examples = train_data + # pass `size` only to learners that take it (model selection); the + # plain learners used by e.g. compare() have a (dataset)-only signature + h = learner(dataset, size) if size is not None else learner(dataset) + fold_errT += err_ratio(h, dataset, train_data) + fold_errV += err_ratio(h, dataset, val_data) + # reverting back to original once test is completed + dataset.examples = examples + return fold_errT / k, fold_errV / k + + +def leave_one_out(learner, dataset, size=None): + """Leave one out cross-validation over the dataset.""" + return cross_validation(learner, dataset, size, len(dataset.examples)) + + +def learning_curve(learner, dataset, trials=10, sizes=None): + """Return a list of (training-set size, mean accuracy) pairs, obtained by + repeatedly cross-validating the learner on training sets of each given size.""" + if sizes is None: + sizes = list(range(2, len(dataset.examples) - trials, 2)) + + def score(learner, size): + random.shuffle(dataset.examples) + return cross_validation(learner, dataset, size, trials) + + return [(size, mean([score(learner, size) for _ in range(trials)])) for size in sizes] + + +def PluralityLearner(dataset): + """ + A very dumb algorithm: always pick the result that was most popular + in the training data. Makes a baseline for comparison. + """ + most_popular = mode([e[dataset.target] for e in dataset.examples]) + + def predict(example): + """Always return same result: the most popular from the training set.""" + return most_popular + + return predict + + +class DecisionFork: + """ + A fork of a decision tree holds an attribute to test, and a dict + of branches, one for each of the attribute's values. + """ + + def __init__(self, attr, attr_name=None, default_child=None, branches=None): + """Initialize by saying what attribute this node tests.""" + self.attr = attr + self.attr_name = attr_name or attr + self.default_child = default_child + self.branches = branches or {} + + def __call__(self, example): + """Given an example, classify it using the attribute and the branches.""" + attr_val = example[self.attr] + if attr_val in self.branches: + return self.branches[attr_val](example) + else: + # return default class when attribute is unknown + return self.default_child(example) + + def add(self, val, subtree): + """Add a branch. If self.attr = val, go to the given subtree.""" + self.branches[val] = subtree + + def display(self, indent=0): + """Print this subtree, showing the tested attribute and each branch, indented by ``indent``.""" + name = self.attr_name + print('Test', name) + for (val, subtree) in self.branches.items(): + print(' ' * 4 * indent, name, '=', val, '==>', end=' ') + subtree.display(indent + 1) + + def __repr__(self): + return 'DecisionFork({0!r}, {1!r}, {2!r})'.format(self.attr, self.attr_name, self.branches) + + +class DecisionLeaf: + """A leaf of a decision tree holds just a result.""" + + def __init__(self, result): + self.result = result + + def __call__(self, example): + return self.result + + def display(self): + """Print the result stored at this leaf.""" + print('RESULT =', self.result) + + def __repr__(self): + return repr(self.result) + + +def DecisionTreeLearner(dataset): + """[Figure 18.5] Learn a decision tree by recursively splitting the examples on + the attribute with the highest information gain, until each branch is pure (or + no attributes remain, falling back to the plurality class).""" + + target, values = dataset.target, dataset.values + + def decision_tree_learning(examples, attrs, parent_examples=()): + if len(examples) == 0: + return plurality_value(parent_examples) + if all_same_class(examples): + return DecisionLeaf(examples[0][target]) + if len(attrs) == 0: + return plurality_value(examples) + A = choose_attribute(attrs, examples) + tree = DecisionFork(A, dataset.attr_names[A], plurality_value(examples)) + for (v_k, exs) in split_by(A, examples): + subtree = decision_tree_learning(exs, remove_all(A, attrs), examples) + tree.add(v_k, subtree) + return tree + + def plurality_value(examples): + """ + Return the most popular target value for this set of examples. + (If target is binary, this is the majority; otherwise plurality). + """ + popular = argmax_random_tie(values[target], key=lambda v: count(target, v, examples)) + return DecisionLeaf(popular) + + def count(attr, val, examples): + """Count the number of examples that have example[attr] = val.""" + return sum(e[attr] == val for e in examples) + + def all_same_class(examples): + """Are all these examples in the same target class?""" + class0 = examples[0][target] + return all(e[target] == class0 for e in examples) + + def choose_attribute(attrs, examples): + """Choose the attribute with the highest information gain.""" + return argmax_random_tie(attrs, key=lambda a: information_gain(a, examples)) + + def information_gain(attr, examples): + """Return the expected reduction in entropy from splitting by attr.""" + + def I(examples): + return information_content([count(target, v, examples) for v in values[target]]) + + n = len(examples) + remainder = sum((len(examples_i) / n) * I(examples_i) for (v, examples_i) in split_by(attr, examples)) + return I(examples) - remainder + + def split_by(attr, examples): + """Return a list of (val, examples) pairs for each val of attr.""" + return [(v, [e for e in examples if e[attr] == v]) for v in values[attr]] + + return decision_tree_learning(dataset.examples, dataset.inputs) + + +def information_content(values): + """Number of bits to represent the probability distribution in values.""" + probabilities = normalize(remove_all(0, values)) + return sum(-p * np.log2(p) for p in probabilities) + + +def DecisionListLearner(dataset, max_test_size=None): + """ + [Figure 18.11] + A decision list is a list of (test, outcome) pairs, where a test is a + conjunction of (attribute, value) literals. An example is classified by the + outcome of the first test it satisfies. Learning repeatedly finds a test that + selects a non-empty subset of the remaining examples that all share a single + outcome, appends (test, outcome), and removes those examples (Figure 18.11). + Works on datasets with discrete attribute values. + """ + attrs = dataset.inputs + target = dataset.target + # the largest conjunction tried; the default lets a test pin down a single + # example, so a consistent list exists whenever the data is not contradictory + max_size = max_test_size or len(attrs) + + def passes(example, test): + """Does the example satisfy every literal of the (conjunctive) test?""" + return all(example[attr] == val for attr, val in test) + + def find_examples(examples): + """Find the smallest test selecting a non-empty subset of examples that + all share one outcome; return (test, outcome, matched_examples), or + (None, None, None) if no such test exists up to max_size literals.""" + literals = sorted({(attr, e[attr]) for e in examples for attr in attrs}, key=str) + for size in range(1, max_size + 1): + for test in itertools.combinations(literals, size): + # a test may constrain each attribute at most once + if len({attr for attr, _ in test}) != size: + continue + matched = [e for e in examples if passes(e, test)] + if matched and len({e[target] for e in matched}) == 1: + return test, matched[0][target], matched + return None, None, None + + def decision_list_learning(examples): + if not examples: + return [((), None)] # catch-all: the empty test matches any example + test, outcome, matched = find_examples(examples) + if test is None: + raise ValueError('DecisionListLearner: examples are not separable ' + '(contradictory examples sharing identical attributes)') + return [(test, outcome)] + decision_list_learning([e for e in examples if e not in matched]) + + def predict(example): + """Return the outcome of the first test the example satisfies.""" + for test, outcome in predict.decision_list: + if passes(example, test): + return outcome + + predict.decision_list = decision_list_learning(list(dataset.examples)) + + return predict + + +def NearestNeighborLearner(dataset, k=1): + """k-NearestNeighbor: the k nearest neighbors vote.""" + + def predict(example): + """Find the k closest items, and have them vote for the best.""" + # the enumerate() index is a tiebreaker so that equal-distance examples + # are never compared directly (examples may be numpy arrays, e.g. MNIST + # images, which are not orderable -> "truth value is ambiguous") + best = heapq.nsmallest(k, ((dataset.distance(e, example), i, e) + for i, e in enumerate(dataset.examples))) + return mode(e[dataset.target] for (d, i, e) in best) + + return predict + + +def LinearLearner(dataset, learning_rate=0.01, epochs=100): + """ + [Section 18.6.3] + Linear classifier with hard threshold. + """ + idx_i = dataset.inputs + idx_t = dataset.target + examples = dataset.examples + num_examples = len(examples) + + # X transpose: the actual value of each input feature across the examples + X_col = [[example[i] for example in examples] for i in idx_i] # vertical columns of X + + # add dummy + ones = [1 for _ in range(len(examples))] + X_col = [ones] + X_col + + # initialize random weights + num_weights = len(idx_i) + 1 + w = random_weights(min_value=-0.5, max_value=0.5, num_weights=num_weights) + + for epoch in range(epochs): + err = [] + # pass over all examples + for example in examples: + x = [1] + [example[i] for i in idx_i] + y = np.dot(w, x) + t = example[idx_t] + err.append(t - y) + + # update weights + for i in range(len(w)): + w[i] = w[i] + learning_rate * (np.dot(err, X_col[i]) / num_examples) + + def predict(example): + x = [1] + [example[i] for i in idx_i] + return np.dot(w, x) + + return predict + + +def LogisticLinearLeaner(dataset, learning_rate=0.01, epochs=100): + """ + [Section 18.6.4] + Linear classifier with logistic regression. + """ + idx_i = dataset.inputs + idx_t = dataset.target + examples = dataset.examples + num_examples = len(examples) + + # X transpose: the actual value of each input feature across the examples + X_col = [[example[i] for example in examples] for i in idx_i] # vertical columns of X + + # add dummy + ones = [1 for _ in range(len(examples))] + X_col = [ones] + X_col + + # initialize random weights + num_weights = len(idx_i) + 1 + w = random_weights(min_value=-0.5, max_value=0.5, num_weights=num_weights) + + for epoch in range(epochs): + err = [] + h = [] + # pass over all examples + for example in examples: + x = [1] + [example[i] for i in idx_i] + y = sigmoid(np.dot(w, x)) + h.append(sigmoid_derivative(y)) + t = example[idx_t] + err.append(t - y) + + # update weights + for i in range(len(w)): + buffer = [x * y for x, y in zip(err, h)] + w[i] = w[i] + learning_rate * (np.dot(buffer, X_col[i]) / num_examples) + + def predict(example): + x = [1] + [example[i] for i in idx_i] + return sigmoid(np.dot(w, x)) + + return predict + + +def NeuralNetLearner(dataset, hidden_layer_sizes=None, learning_rate=0.01, epochs=100, activation=sigmoid): + """ + Layered feed-forward network. + hidden_layer_sizes: List of number of hidden units per hidden layer + learning_rate: Learning rate of gradient descent + epochs: Number of passes over the dataset + """ + + if hidden_layer_sizes is None: + hidden_layer_sizes = [3] + i_units = len(dataset.inputs) + o_units = len(dataset.values[dataset.target]) + + # construct a network + raw_net = network(i_units, hidden_layer_sizes, o_units, activation) + learned_net = BackPropagationLearner(dataset, raw_net, learning_rate, epochs, activation) + + def predict(example): + # input nodes + i_nodes = learned_net[0] + + # activate input layer + for v, n in zip(example, i_nodes): + n.value = v + + # forward pass + for layer in learned_net[1:]: + for node in layer: + inc = [n.value for n in node.inputs] + in_val = dot_product(inc, node.weights) + node.value = node.activation(in_val) + + # hypothesis + o_nodes = learned_net[-1] + prediction = find_max_node(o_nodes) + return prediction + + return predict + + +def BackPropagationLearner(dataset, net, learning_rate, epochs, activation=sigmoid): + """ + [Figure 18.23] + The back-propagation algorithm for multilayer networks. + """ + # initialise weights + for layer in net: + for node in layer: + node.weights = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(node.weights)) + + examples = dataset.examples + # As of now dataset.target gives an int instead of list, + # Changing dataset class will have effect on all the learners. + # Will be taken care of later. + o_nodes = net[-1] + i_nodes = net[0] + o_units = len(o_nodes) + idx_t = dataset.target + idx_i = dataset.inputs + n_layers = len(net) + + inputs, targets = init_examples(examples, idx_i, idx_t, o_units) + + for epoch in range(epochs): + # iterate over each example + for e in range(len(examples)): + i_val = inputs[e] + t_val = targets[e] + + # activate input layer + for v, n in zip(i_val, i_nodes): + n.value = v + + # forward pass + for layer in net[1:]: + for node in layer: + inc = [n.value for n in node.inputs] + in_val = dot_product(inc, node.weights) + node.value = node.activation(in_val) + + # initialize delta + delta = [[] for _ in range(n_layers)] + + # compute outer layer delta + + # error for the MSE cost function + err = [t_val[i] - o_nodes[i].value for i in range(o_units)] + + # calculate delta at output + if node.activation == sigmoid: + delta[-1] = [sigmoid_derivative(o_nodes[i].value) * err[i] for i in range(o_units)] + elif node.activation == relu: + delta[-1] = [relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)] + elif node.activation == tanh: + delta[-1] = [tanh_derivative(o_nodes[i].value) * err[i] for i in range(o_units)] + elif node.activation == elu: + delta[-1] = [elu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)] + elif node.activation == leaky_relu: + delta[-1] = [leaky_relu_derivative(o_nodes[i].value) * err[i] for i in range(o_units)] + else: + return ValueError("Activation function unknown.") + + # backward pass + h_layers = n_layers - 2 + for i in range(h_layers, 0, -1): + layer = net[i] + h_units = len(layer) + nx_layer = net[i + 1] + + # weights from each ith layer node to each i + 1th layer node + w = [[node.weights[k] for node in nx_layer] for k in range(h_units)] + + if activation == sigmoid: + delta[i] = [sigmoid_derivative(layer[j].value) * dot_product(w[j], delta[i + 1]) + for j in range(h_units)] + elif activation == relu: + delta[i] = [relu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1]) + for j in range(h_units)] + elif activation == tanh: + delta[i] = [tanh_derivative(layer[j].value) * dot_product(w[j], delta[i + 1]) + for j in range(h_units)] + elif activation == elu: + delta[i] = [elu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1]) + for j in range(h_units)] + elif activation == leaky_relu: + delta[i] = [leaky_relu_derivative(layer[j].value) * dot_product(w[j], delta[i + 1]) + for j in range(h_units)] + else: + return ValueError("Activation function unknown.") + + # update weights + for i in range(1, n_layers): + layer = net[i] + inc = [node.value for node in net[i - 1]] + units = len(layer) + for j in range(units): + layer[j].weights = vector_add(layer[j].weights, + scalar_vector_product(learning_rate * delta[i][j], inc)) + + return net + + +def PerceptronLearner(dataset, learning_rate=0.01, epochs=100): + """Logistic Regression, NO hidden layer""" + i_units = len(dataset.inputs) + o_units = len(dataset.values[dataset.target]) + hidden_layer_sizes = [] + raw_net = network(i_units, hidden_layer_sizes, o_units) + learned_net = BackPropagationLearner(dataset, raw_net, learning_rate, epochs) + + def predict(example): + o_nodes = learned_net[1] + + # forward pass + for node in o_nodes: + in_val = dot_product(example, node.weights) + node.value = node.activation(in_val) + + # hypothesis + return find_max_node(o_nodes) + + return predict + + +class NNUnit: + """ + Single Unit of Multiple Layer Neural Network + inputs: Incoming connections + weights: Weights to incoming connections + """ + + def __init__(self, activation=sigmoid, weights=None, inputs=None): + self.weights = weights or [] + self.inputs = inputs or [] + self.value = None + self.activation = activation + + +def network(input_units, hidden_layer_sizes, output_units, activation=sigmoid): + """ + Create Directed Acyclic Network of given number layers. + hidden_layers_sizes : List number of neuron units in each hidden layer + excluding input and output layers + """ + layers_sizes = [input_units] + hidden_layer_sizes + [output_units] + + net = [[NNUnit(activation) for _ in range(size)] for size in layers_sizes] + n_layers = len(net) + + # make connection + for i in range(1, n_layers): + for n in net[i]: + for k in net[i - 1]: + n.inputs.append(k) + n.weights.append(0) + return net + + +def init_examples(examples, idx_i, idx_t, o_units): + """Split examples into input and target dicts keyed by example index. + + Inputs are read from the attribute positions in ``idx_i`` and targets from + position ``idx_t``. When ``o_units`` > 1 each target is one-hot encoded over + ``o_units`` units, otherwise it is wrapped in a single-element list. Returns + the pair (inputs, targets).""" + inputs, targets = {}, {} + + for i, e in enumerate(examples): + # input values of e + inputs[i] = [e[i] for i in idx_i] + + if o_units > 1: + # one-hot representation of e's target + t = [0 for i in range(o_units)] + t[e[idx_t]] = 1 + targets[i] = t + else: + # target value of e + targets[i] = [e[idx_t]] + + return inputs, targets + + +def find_max_node(nodes): + """Return the index of the node with the greatest ``value`` attribute.""" + return nodes.index(max(nodes, key=lambda node: node.value)) + + +class SVC: + """Support Vector Classifier trained in dual form by solving a quadratic + programming problem; supports arbitrary kernels and a soft-margin penalty ``C``.""" + + def __init__(self, kernel=linear_kernel, C=1.0, verbose=False): + self.kernel = kernel + self.C = C # hyper-parameter + self.sv_idx, self.sv, self.sv_y = np.zeros(0), np.zeros(0), np.zeros(0) + self.alphas = np.zeros(0) + self.w = None + self.b = 0.0 # intercept + self.verbose = verbose + + def fit(self, X, y): + """ + Trains the model by solving a quadratic programming problem. + :param X: array of size [n_samples, n_features] holding the training samples + :param y: array of size [n_samples] holding the class labels + """ + # In QP formulation (dual): m variables, 2m+1 constraints (1 equation, 2m inequations) + self.solve_qp(X, y) + sv = self.alphas > 1e-5 + self.sv_idx = np.arange(len(self.alphas))[sv] + self.sv, self.sv_y, self.alphas = X[sv], y[sv], self.alphas[sv] + + if self.kernel == linear_kernel: + self.w = np.dot(self.alphas * self.sv_y, self.sv) + + for n in range(len(self.alphas)): + self.b += self.sv_y[n] + self.b -= np.sum(self.alphas * self.sv_y * self.K[self.sv_idx[n], sv]) + self.b /= len(self.alphas) + return self + + def solve_qp(self, X, y): + """ + Solves a quadratic programming problem. In QP formulation (dual): + m variables, 2m+1 constraints (1 equation, 2m inequations). + :param X: array of size [n_samples, n_features] holding the training samples + :param y: array of size [n_samples] holding the class labels + """ + m = len(y) # m = n_samples + self.K = self.kernel(X) # gram matrix + P = self.K * np.outer(y, y) + q = -np.ones(m) + lb = np.zeros(m) # lower bounds + ub = np.ones(m) * self.C # upper bounds + A = y.astype(np.float64) # equality matrix + b = np.zeros(1) # equality vector + # imported lazily: SVM training is the only thing that needs the optional + # qpsolvers/cvxopt dependency, so the rest of the module (trees, kNN, naive + # Bayes, neural nets, ...) stays importable without it + from qpsolvers import solve_qp + self.alphas = solve_qp(P, q, A=A, b=b, lb=lb, ub=ub, solver='cvxopt', + sym_proj=True, verbose=self.verbose) + + def predict_score(self, X): + """ + Predicts the score for a given example. + """ + if self.w is None: + return np.dot(self.alphas * self.sv_y, self.kernel(self.sv, X)) + self.b + return np.dot(X, self.w) + self.b + + def predict(self, X): + """ + Predicts the class of a given example. + """ + return np.sign(self.predict_score(X)) + + +class SVR: + """Support Vector Regressor trained in dual form by solving a quadratic + programming problem, using an epsilon-insensitive loss and penalty ``C``.""" + + def __init__(self, kernel=linear_kernel, C=1.0, epsilon=0.1, verbose=False): + self.kernel = kernel + self.C = C # hyper-parameter + self.epsilon = epsilon # epsilon insensitive loss value + self.sv_idx, self.sv = np.zeros(0), np.zeros(0) + self.alphas_p, self.alphas_n = np.zeros(0), np.zeros(0) + self.w = None + self.b = 0.0 # intercept + self.verbose = verbose + + def fit(self, X, y): + """ + Trains the model by solving a quadratic programming problem. + :param X: array of size [n_samples, n_features] holding the training samples + :param y: array of size [n_samples] holding the class labels + """ + # In QP formulation (dual): m variables, 2m+1 constraints (1 equation, 2m inequations) + self.solve_qp(X, y) + + sv = np.logical_or(self.alphas_p > 1e-5, self.alphas_n > 1e-5) + self.sv_idx = np.arange(len(self.alphas_p))[sv] + self.sv, sv_y = X[sv], y[sv] + self.alphas_p, self.alphas_n = self.alphas_p[sv], self.alphas_n[sv] + + if self.kernel == linear_kernel: + self.w = np.dot(self.alphas_p - self.alphas_n, self.sv) + + for n in range(len(self.alphas_p)): + self.b += sv_y[n] + self.b -= np.sum((self.alphas_p - self.alphas_n) * self.K[self.sv_idx[n], sv]) + self.b -= self.epsilon + self.b /= len(self.alphas_p) + + return self + + def solve_qp(self, X, y): + """ + Solves a quadratic programming problem. In QP formulation (dual): + m variables, 2m+1 constraints (1 equation, 2m inequations). + :param X: array of size [n_samples, n_features] holding the training samples + :param y: array of size [n_samples] holding the class labels + """ + # + m = len(y) # m = n_samples + self.K = self.kernel(X) # gram matrix + P = np.vstack((np.hstack((self.K, -self.K)), # alphas_p, alphas_n + np.hstack((-self.K, self.K)))) # alphas_n, alphas_p + q = np.hstack((-y, y)) + self.epsilon + lb = np.zeros(2 * m) # lower bounds + ub = np.ones(2 * m) * self.C # upper bounds + A = np.hstack((np.ones(m), -np.ones(m))) # equality matrix + b = np.zeros(1) # equality vector + from qpsolvers import solve_qp # lazy: see SVC.solve_qp above + alphas = solve_qp(P, q, A=A, b=b, lb=lb, ub=ub, solver='cvxopt', + sym_proj=True, verbose=self.verbose) + self.alphas_p = alphas[:m] + self.alphas_n = alphas[m:] + + def predict(self, X): + """Predict the regression target value(s) for the samples ``X``.""" + if self.kernel != linear_kernel: + return np.dot(self.alphas_p - self.alphas_n, self.kernel(self.sv, X)) + self.b + return np.dot(X, self.w) + self.b + + +class MultiClassLearner: + """Wrap a binary classifier ``clf`` to handle multiple classes, using either + the one-vs-rest ('ovr') or one-vs-one ('ovo') decision function.""" + + def __init__(self, clf, decision_function='ovr'): + self.clf = clf + self.decision_function = decision_function + self.n_class, self.classifiers = 0, [] + + def fit(self, X, y): + """ + Trains n_class or n_class * (n_class - 1) / 2 classifiers + according to the training method, ovr or ovo respectively. + :param X: array of size [n_samples, n_features] holding the training samples + :param y: array of size [n_samples] holding the class labels + :return: array of classifiers + """ + labels = np.unique(y) + self.n_class = len(labels) + if self.decision_function == 'ovr': # one-vs-rest method + for label in labels: + y1 = np.array(y) + y1[y1 != label] = -1.0 + y1[y1 == label] = 1.0 + self.clf.fit(X, y1) + self.classifiers.append(copy.deepcopy(self.clf)) + elif self.decision_function == 'ovo': # use one-vs-one method + n_labels = len(labels) + for i in range(n_labels): + for j in range(i + 1, n_labels): + neg_id, pos_id = y == labels[i], y == labels[j] + X1, y1 = np.r_[X[neg_id], X[pos_id]], np.r_[y[neg_id], y[pos_id]] + y1[y1 == labels[i]] = -1.0 + y1[y1 == labels[j]] = 1.0 + self.clf.fit(X1, y1) + self.classifiers.append(copy.deepcopy(self.clf)) + else: + return ValueError("Decision function must be either 'ovr' or 'ovo'.") + return self + + def predict(self, X): + """ + Predicts the class of a given example according to the training method. + """ + n_samples = len(X) + if self.decision_function == 'ovr': # one-vs-rest method + assert len(self.classifiers) == self.n_class + score = np.zeros((n_samples, self.n_class)) + for i in range(self.n_class): + clf = self.classifiers[i] + score[:, i] = clf.predict_score(X) + return np.argmax(score, axis=1) + elif self.decision_function == 'ovo': # use one-vs-one method + assert len(self.classifiers) == self.n_class * (self.n_class - 1) / 2 + vote = np.zeros((n_samples, self.n_class)) + clf_id = 0 + for i in range(self.n_class): + for j in range(i + 1, self.n_class): + res = self.classifiers[clf_id].predict(X) + vote[res < 0, i] += 1.0 # negative sample: class i + vote[res > 0, j] += 1.0 # positive sample: class j + clf_id += 1 + return np.argmax(vote, axis=1) + else: + return ValueError("Decision function must be either 'ovr' or 'ovo'.") + + +def EnsembleLearner(learners): + """Given a list of learning algorithms, have them vote.""" + + def train(dataset): + predictors = [learner(dataset) for learner in learners] + + def predict(example): + return mode(predictor(example) for predictor in predictors) + + return predict + + return train + + +def ada_boost(dataset, L, K): + """[Figure 18.34] AdaBoost: train ``K`` hypotheses with the weighted learner + ``L``, increasing the weight of misclassified examples after each round, and + return their weighted-majority ensemble.""" + + examples, target = dataset.examples, dataset.target + n = len(examples) + eps = 1 / (2 * n) + w = [1 / n] * n + h, z = [], [] + for k in range(K): + h_k = L(dataset, w) + h.append(h_k) + error = sum(weight for example, weight in zip(examples, w) if example[target] != h_k(example)) + # avoid divide-by-0 from either 0% or 100% error rates + error = np.clip(error, eps, 1 - eps) + for j, example in enumerate(examples): + if example[target] == h_k(example): + w[j] *= error / (1 - error) + w = normalize(w) + z.append(np.log((1 - error) / error)) + return weighted_majority(h, z) + + +def weighted_majority(predictors, weights): + """Return a predictor that takes a weighted vote.""" + + def predict(example): + return weighted_mode((predictor(example) for predictor in predictors), weights) + + return predict + + +def weighted_mode(values, weights): + """ + Return the value with the greatest total weight. + >>> weighted_mode('abbaa', [1, 2, 3, 1, 2]) + 'b' + """ + totals = defaultdict(int) + for v, w in zip(values, weights): + totals[v] += w + return max(totals, key=totals.__getitem__) + + +def RandomForest(dataset, n=5): + """An ensemble of Decision Trees trained using bagging and feature bagging.""" + + def data_bagging(dataset, m=0): + """Sample m examples with replacement""" + n = len(dataset.examples) + return weighted_sample_with_replacement(m or n, dataset.examples, [1] * n) + + def feature_bagging(dataset, p=0.7): + """Feature bagging with probability p to retain an attribute""" + inputs = [i for i in dataset.inputs if probability(p)] + return inputs or dataset.inputs + + def predict(example): + print([predictor(example) for predictor in predictors]) + return mode(predictor(example) for predictor in predictors) + + predictors = [DecisionTreeLearner(DataSet(examples=data_bagging(dataset), attrs=dataset.attrs, + attr_names=dataset.attr_names, target=dataset.target, + inputs=feature_bagging(dataset))) for _ in range(n)] + + return predict + + +def WeightedLearner(unweighted_learner): + """ + [Page 749 footnote 14] + Given a learner that takes just an unweighted dataset, return + one that takes also a weight for each example. + """ + + def train(dataset, weights): + return unweighted_learner(replicated_dataset(dataset, weights)) + + return train + + +def replicated_dataset(dataset, weights, n=None): + """Copy dataset, replicating each example in proportion to its weight.""" + n = n or len(dataset.examples) + result = copy.copy(dataset) + result.examples = weighted_replicate(dataset.examples, weights, n) + return result + + +def weighted_replicate(seq, weights, n): + """ + Return n selections from seq, with the count of each element of + seq proportional to the corresponding weight (filling in fractions + randomly). + >>> weighted_replicate('ABC', [1, 2, 1], 4) + ['A', 'B', 'B', 'C'] + """ + assert len(seq) == len(weights) + weights = normalize(weights) + wholes = [int(w * n) for w in weights] + fractions = [(w * n) % 1 for w in weights] + return (flatten([x] * nx for x, nx in zip(seq, wholes)) + + weighted_sample_with_replacement(n - sum(wholes), seq, fractions)) + + +# metrics + +def accuracy_score(y_pred, y_true): + """Return the fraction of predictions in ``y_pred`` that match ``y_true``.""" + assert y_pred.shape == y_true.shape + return np.mean(y_pred == y_true) + + +def r2_score(y_pred, y_true): + """Return the R^2 (coefficient of determination) of ``y_pred`` against ``y_true``.""" + assert y_pred.shape == y_true.shape + return 1. - (np.sum(np.square(y_pred - y_true)) / # sum of square of residuals + np.sum(np.square(y_true - np.mean(y_true)))) # total sum of squares + + +# datasets +# Loaded from the aima-data directory at import time. Guard against it being +# absent (e.g. in a browser/Pyodide environment) so the rest of the module still +# imports; these names are None when the data directory is unavailable. +try: + orings = DataSet(name='orings', target='Distressed', attr_names='Rings Distressed Temp Pressure Flightnum') + + zoo = DataSet(name='zoo', target='type', exclude=['name'], + attr_names='name hair feathers eggs milk airborne aquatic predator toothed backbone ' + 'breathes venomous fins legs tail domestic catsize type') + + iris = DataSet(name='iris', target='class', attr_names='sepal-len sepal-width petal-len petal-width class') +except FileNotFoundError: + orings = zoo = iris = None + + +def RestaurantDataSet(examples=None): + """ + [Figure 18.3] + Build a DataSet of Restaurant waiting examples. + """ + return DataSet(name='restaurant', target='Wait', examples=examples, + attr_names='Alternate Bar Fri/Sat Hungry Patrons Price Raining Reservation Type WaitEstimate Wait') + + +try: + restaurant = RestaurantDataSet() +except FileNotFoundError: + restaurant = None + + +def T(attr_name, branches): + """Build a DecisionFork testing the restaurant attribute ``attr_name``, wrapping each + non-fork child in a DecisionLeaf; a shorthand for writing decision trees by hand.""" + branches = {value: (child if isinstance(child, DecisionFork) else DecisionLeaf(child)) + for value, child in branches.items()} + return DecisionFork(restaurant.attr_num(attr_name), attr_name, print, branches) + + +""" +[Figure 18.2] +A decision tree for deciding whether to wait for a table at a hotel. +""" + +waiting_decision_tree = None if restaurant is None else T('Patrons', + {'None': 'No', 'Some': 'Yes', + 'Full': T('WaitEstimate', + {'>60': 'No', '0-10': 'Yes', + '30-60': T('Alternate', + {'No': T('Reservation', + {'Yes': 'Yes', + 'No': T('Bar', {'No': 'No', + 'Yes': 'Yes'})}), + 'Yes': T('Fri/Sat', {'No': 'No', 'Yes': 'Yes'})}), + '10-30': T('Hungry', + {'No': 'Yes', + 'Yes': T('Alternate', + {'No': 'Yes', + 'Yes': T('Raining', + {'No': 'No', + 'Yes': 'Yes'})})})})}) + + +def SyntheticRestaurant(n=20): + """Generate a DataSet with n examples.""" + + def gen(): + example = list(map(random.choice, restaurant.values)) + example[restaurant.target] = waiting_decision_tree(example) + return example + + return RestaurantDataSet([gen() for _ in range(n)]) + + +def Majority(k, n): + """ + Return a DataSet with n k-bit examples of the majority problem: + k random bits followed by a 1 if more than half the bits are 1, else 0. + """ + examples = [] + for i in range(n): + bits = [random.choice([0, 1]) for _ in range(k)] + bits.append(int(sum(bits) > k / 2)) + examples.append(bits) + return DataSet(name='majority', examples=examples) + + +def Parity(k, n, name='parity'): + """ + Return a DataSet with n k-bit examples of the parity problem: + k random bits followed by a 1 if an odd number of bits are 1, else 0. + """ + examples = [] + for i in range(n): + bits = [random.choice([0, 1]) for _ in range(k)] + bits.append(sum(bits) % 2) + examples.append(bits) + return DataSet(name=name, examples=examples) + + +def Xor(n): + """Return a DataSet with n examples of 2-input xor.""" + return Parity(2, n, name='xor') + + +def ContinuousXor(n): + """2 inputs are chosen uniformly from (0.0 .. 2.0]; output is xor of ints.""" + examples = [] + for i in range(n): + x, y = [random.uniform(0.0, 2.0) for _ in '12'] + examples.append([x, y, x != y]) + return DataSet(name='continuous xor', examples=examples) + + +def gaussian_mixture_em(dataset, k, epsilon=1e-4, max_iterations=100): + """ + [Section 20.3] + Unsupervised clustering with the Expectation-Maximization (EM) algorithm, + fitting a mixture of k Gaussians to 'dataset' (a sequence of points). Each + iteration performs two steps:: + + E-step: compute the responsibilities p_ij = P(C=i | x_j), the posterior + probability that point x_j was generated by component i, which by + Bayes' rule is proportional to P(x_j | C=i) * P(C=i). + M-step: re-estimate the weight, mean and covariance of each component as + the responsibility-weighted statistics of the whole data set. + + EM is guaranteed to increase the data log likelihood at each iteration; it is + iterated until that improvement falls below 'epsilon' or 'max_iterations' is + reached. Returns a dict with the fitted 'weights', 'means', 'covariances' and + the final 'responsibilities'. + """ + X = np.asarray(dataset, dtype=float) + n, d = X.shape + + def multivariate_gaussian(points, mean, cov): + """Density of N(mean, cov) evaluated at each row of 'points'.""" + diff = points - mean + return (np.exp(-0.5 * np.sum(diff @ np.linalg.inv(cov) * diff, axis=1)) / + np.sqrt((2 * np.pi) ** d * np.linalg.det(cov))) + + # initialize: uniform weights, means at k distinct random data points and + # covariances at the sample covariance of the whole data set + weights = np.full(k, 1 / k) + means = X[np.random.choice(n, k, replace=False)] + covariances = np.array([np.cov(X, rowvar=False) for _ in range(k)]) + + log_likelihood = -np.inf + responsibilities = np.zeros((n, k)) + for _ in range(max_iterations): + # E-step: p_ij = alpha * P(x_j | C=i) * P(C=i) + for i in range(k): + responsibilities[:, i] = weights[i] * multivariate_gaussian(X, means[i], covariances[i]) + point_likelihoods = responsibilities.sum(axis=1) + responsibilities /= point_likelihoods[:, np.newaxis] + + # M-step: refit each component to the responsibility-weighted data + counts = responsibilities.sum(axis=0) + weights = counts / n + means = (responsibilities.T @ X) / counts[:, np.newaxis] + for i in range(k): + diff = X - means[i] + # regularize the covariance to avoid the degenerate zero-variance maximum + covariances[i] = (responsibilities[:, i] * diff.T) @ diff / counts[i] + 1e-6 * np.eye(d) + + # stop once the data log likelihood stops improving appreciably + new_log_likelihood = np.sum(np.log(point_likelihoods)) + if abs(new_log_likelihood - log_likelihood) < epsilon: + break + log_likelihood = new_log_likelihood + + return {'weights': weights, 'means': means, 'covariances': covariances, + 'responsibilities': responsibilities} + + +def naive_bayes_em(dataset, k, epsilon=1e-4, max_iterations=100): + """ + [Section 20.3] + Learn the parameters of a Bayes net with a hidden variable via EM: a naive + Bayes model with a hidden k-valued class (the 'bags of candy' example of + Section 20.3.2). 'dataset' is a sequence of binary feature vectors; given the + unobserved class, the features are independent Bernoulli variables. Each + iteration performs two steps:: + + E-step: responsibilities r_ji = P(class=i | x_j), which by Bayes' rule and + conditional independence is proportional to + P(class=i) * prod_f P(x_jf | class=i). + M-step: re-estimate the class priors and every conditional probability + P(feature_f = 1 | class=i) as the responsibility-weighted counts. + + Returns a dict with the learned class 'weights', the 'probabilities' matrix + (k x d, entry [i][f] = P(feature f = 1 | class i)) and the final + 'responsibilities'. + """ + X = np.asarray(dataset, dtype=float) + n, d = X.shape + + # initialize: uniform priors and random conditionals (a symmetric init is a + # fixed point of EM, so the components must start out distinct) + weights = np.full(k, 1 / k) + theta = np.random.uniform(0.25, 0.75, size=(k, d)) + + log_likelihood = -np.inf + responsibilities = np.zeros((n, k)) + for _ in range(max_iterations): + # E-step: r_ji = alpha * P(class=i) * prod_f theta_if^x_jf (1-theta_if)^(1-x_jf) + for i in range(k): + responsibilities[:, i] = weights[i] * np.prod(theta[i] ** X * (1 - theta[i]) ** (1 - X), axis=1) + point_likelihoods = responsibilities.sum(axis=1) + responsibilities /= point_likelihoods[:, np.newaxis] + + # M-step: priors and conditional probabilities from the expected counts + counts = responsibilities.sum(axis=0) + weights = counts / n + theta = (responsibilities.T @ X) / counts[:, np.newaxis] + # keep the probabilities inside (0, 1) to avoid 0/0 in the next E-step + theta = np.clip(theta, 1e-9, 1 - 1e-9) + + # stop once the data log likelihood stops improving appreciably + new_log_likelihood = np.sum(np.log(point_likelihoods)) + if abs(new_log_likelihood - log_likelihood) < epsilon: + break + log_likelihood = new_log_likelihood + + return {'weights': weights, 'probabilities': theta, 'responsibilities': responsibilities} + + +def compare(algorithms=None, datasets=None, k=10, trials=1): + """ + Compare various learners on various datasets using cross-validation. + Print results as a table. + """ + # default list of algorithms + algorithms = algorithms or [PluralityLearner, NaiveBayesLearner, NearestNeighborLearner, DecisionTreeLearner] + + # default list of datasets + datasets = datasets or [iris, orings, zoo, restaurant, SyntheticRestaurant(20), + Majority(7, 100), Parity(7, 100), Xor(100)] + + print_table([[a.__name__.replace('Learner', '')] + [cross_validation(a, d, k=k, trials=trials) for d in datasets] + for a in algorithms], header=[''] + [d.name[0:7] for d in datasets], numfmt='%.2f') diff --git a/q-learning-9b/aima/logic.py b/q-learning-9b/aima/logic.py new file mode 100644 index 0000000..700e247 --- /dev/null +++ b/q-learning-9b/aima/logic.py @@ -0,0 +1,2286 @@ +""" +Representations and Inference for Logic. (Chapters 7-9, 12) + +Covers both Propositional and First-Order Logic. First we have four +important data types: + + KB Abstract class holds a knowledge base of logical expressions + KB_Agent Abstract class subclasses agents.Agent + Expr A logical expression, imported from utils.py + substitution Implemented as a dictionary of var:value pairs, {x:1, y:x} + +Be careful: some functions take an Expr as argument, and some take a KB. + +Logical expressions can be created with Expr, imported from utils, or with expr, +which adds the capability to write a string that uses +the connectives ==>, <==, <=>, or <=/=>. But be careful: these have the +operator precedence of commas; you may need to add parens to make precedence work. +See logic.ipynb for examples. + +Then we implement various functions for doing logical inference: + + pl_true Evaluate a propositional logical sentence in a model + tt_entails Say if a statement is entailed by a KB + pl_resolution Do resolution on propositional sentences + dpll_satisfiable See if a propositional sentence is satisfiable + WalkSAT Try to find a solution for a set of clauses + +And a few other functions: + + to_cnf Convert to conjunctive normal form + unify Do unification of two FOL sentences + diff, simp Symbolic differentiation and simplification +""" + +import heapq +import itertools +import random +from collections import defaultdict, Counter + +import networkx as nx + +from aima.agents import Agent, Glitter, Bump, Stench, Breeze, Scream +from aima.csp import parse_neighbors, UniversalDict +from aima.search import astar_search, PlanRoute +from aima.utils import remove_all, unique, first, probability, isnumber, issequence, Expr, expr, subexpressions, extend + + +class KB: + """A knowledge base to which you can tell and ask sentences. + To create a KB, first subclass this class and implement + tell, ask_generator, and retract. Why ask_generator instead of ask? + The book is a bit vague on what ask means -- + For a Propositional Logic KB, ask(P & Q) returns True or False, but for an + FOL KB, something like ask(Brother(x, y)) might return many substitutions + such as {x: Cain, y: Abel}, {x: Abel, y: Cain}, {x: George, y: Jeb}, etc. + So ask_generator generates these one at a time, and ask either returns the + first one or returns False.""" + + def __init__(self, sentence=None): + if sentence: + self.tell(sentence) + + def tell(self, sentence): + """Add the sentence to the KB.""" + raise NotImplementedError + + def ask(self, query): + """Return a substitution that makes the query true, or, failing that, return False.""" + return first(self.ask_generator(query), default=False) + + def ask_generator(self, query): + """Yield all the substitutions that make query true.""" + raise NotImplementedError + + def retract(self, sentence): + """Remove sentence from the KB.""" + raise NotImplementedError + + +class PropKB(KB): + """A KB for propositional logic. Inefficient, with no indexing.""" + + def __init__(self, sentence=None): + super().__init__(sentence) + self.clauses = [] + + def tell(self, sentence): + """Add the sentence's clauses to the KB.""" + self.clauses.extend(conjuncts(to_cnf(sentence))) + + def ask_generator(self, query): + """Yield the empty substitution {} if KB entails query; else no results.""" + if tt_entails(Expr('&', *self.clauses), query): + yield {} + + def ask_if_true(self, query): + """Return True if the KB entails query, else return False.""" + for _ in self.ask_generator(query): + return True + return False + + def retract(self, sentence): + """Remove the sentence's clauses from the KB.""" + for c in conjuncts(to_cnf(sentence)): + if c in self.clauses: + self.clauses.remove(c) + + +# ______________________________________________________________________________ + + +def KBAgentProgram(kb): + """ + [Figure 7.1] + A generic logical knowledge-based agent program. + """ + steps = itertools.count() + + def program(percept): + t = next(steps) + kb.tell(make_percept_sentence(percept, t)) + action = kb.ask(make_action_query(t)) + kb.tell(make_action_sentence(action, t)) + return action + + def make_percept_sentence(percept, t): + return Expr('Percept')(percept, t) + + def make_action_query(t): + return expr('ShouldDo(action, {})'.format(t)) + + def make_action_sentence(action, t): + return Expr('Did')(action[expr('action')], t) + + return program + + +def is_symbol(s): + """A string s is a symbol if it starts with an alphabetic char. + >>> is_symbol('R2D2') + True + """ + return isinstance(s, str) and s[:1].isalpha() + + +def is_var_symbol(s): + """A logic variable symbol is an initial-lowercase string. + >>> is_var_symbol('EXE') + False + """ + return is_symbol(s) and s[0].islower() + + +def is_prop_symbol(s): + """A proposition logic symbol is an initial-uppercase string. + >>> is_prop_symbol('exe') + False + """ + return is_symbol(s) and s[0].isupper() + + +def gensym(prefix='s_'): + """Return a fresh logic symbol with a unique name, like Lisp's ``gensym`` [#230]. + + Successive calls return distinct symbols ``s_0``, ``s_1``, ``s_2``, ... -- handy + for building an array or dict of symbols, or for standardizing apart. Pass a + ``prefix`` to control the name stem. + """ + gensym.counter += 1 + return Expr(prefix + str(gensym.counter)) + + +gensym.counter = -1 + + +def variables(s): + """Return a set of the variables in expression s. + >>> variables(expr('F(x, x) & G(x, y) & H(y, z) & R(A, z, 2)')) == {x, y, z} + True + """ + return {x for x in subexpressions(s) if is_variable(x)} + + +def is_definite_clause(s): + """Returns True for exprs s of the form A & B & ... & C ==> D, + where all literals are positive. In clause form, this is + ~A | ~B | ... | ~C | D, where exactly one clause is positive. + >>> is_definite_clause(expr('Farmer(Mac)')) + True + """ + if is_symbol(s.op): + return True + elif s.op == '==>': + antecedent, consequent = s.args + return is_symbol(consequent.op) and all(is_symbol(arg.op) for arg in conjuncts(antecedent)) + else: + return False + + +def parse_definite_clause(s): + """Return the antecedents and the consequent of a definite clause.""" + assert is_definite_clause(s) + if is_symbol(s.op): + return [], s + else: + antecedent, consequent = s.args + return conjuncts(antecedent), consequent + + +# Useful constant Exprs used in examples and code: +A, B, C, D, E, F, G, P, Q, a, x, y, z, u = map(Expr, 'ABCDEFGPQaxyzu') + + +# ______________________________________________________________________________ + + +def tt_entails(kb, alpha): + """ + [Figure 7.10] + Does kb entail the sentence alpha? Use truth tables. For propositional + kb's and sentences. Note that the 'kb' should be an Expr which is a + conjunction of clauses. + >>> tt_entails(expr('P & Q'), expr('Q')) + True + """ + assert not variables(alpha) + symbols = list(prop_symbols(kb & alpha)) + return tt_check_all(kb, alpha, symbols, {}) + + +def tt_check_all(kb, alpha, symbols, model): + """Auxiliary routine to implement tt_entails.""" + if not symbols: + if pl_true(kb, model): + result = pl_true(alpha, model) + assert result in (True, False) + return result + else: + return True + else: + P, rest = symbols[0], symbols[1:] + return (tt_check_all(kb, alpha, rest, extend(model, P, True)) and + tt_check_all(kb, alpha, rest, extend(model, P, False))) + + +def prop_symbols(x): + """Return the set of all propositional symbols in x.""" + if not isinstance(x, Expr): + return set() + elif is_prop_symbol(x.op): + return {x} + else: + return {symbol for arg in x.args for symbol in prop_symbols(arg)} + + +def constant_symbols(x): + """Return the set of all constant symbols in x.""" + if not isinstance(x, Expr): + return set() + elif is_prop_symbol(x.op) and not x.args: + return {x} + else: + return {symbol for arg in x.args for symbol in constant_symbols(arg)} + + +def predicate_symbols(x): + """Return a set of (symbol_name, arity) in x. + All symbols (even functional) with arity > 0 are considered.""" + if not isinstance(x, Expr) or not x.args: + return set() + pred_set = {(x.op, len(x.args))} if is_prop_symbol(x.op) else set() + pred_set.update({symbol for arg in x.args for symbol in predicate_symbols(arg)}) + return pred_set + + +def tt_true(s): + """Is a propositional sentence a tautology? + >>> tt_true('P | ~P') + True + """ + s = expr(s) + return tt_entails(True, s) + + +def pl_true(exp, model={}): + """Return True if the propositional logic expression is true in the model, + and False if it is false. If the model does not specify the value for + every proposition, this may return None to indicate 'not obvious'; + this may happen even when the expression is tautological. + >>> pl_true(P, {}) is None + True + """ + if exp in (True, False): + return exp + op, args = exp.op, exp.args + if is_prop_symbol(op): + return model.get(exp) + elif op == '~': + p = pl_true(args[0], model) + if p is None: + return None + else: + return not p + elif op == '|': + result = False + for arg in args: + p = pl_true(arg, model) + if p is True: + return True + if p is None: + result = None + return result + elif op == '&': + result = True + for arg in args: + p = pl_true(arg, model) + if p is False: + return False + if p is None: + result = None + return result + p, q = args + if op == '==>': + return pl_true(~p | q, model) + elif op == '<==': + return pl_true(p | ~q, model) + pt = pl_true(p, model) + if pt is None: + return None + qt = pl_true(q, model) + if qt is None: + return None + if op == '<=>': + return pt == qt + elif op == '^': # xor or 'not equivalent' + return pt != qt + else: + raise ValueError('Illegal operator in logic expression' + str(exp)) + + +# ______________________________________________________________________________ + +# Convert to Conjunctive Normal Form (CNF) + + +def to_cnf(s): + """ + [Page 253] + Convert a propositional logical sentence to conjunctive normal form. + That is, to the form ((A | ~B | ...) & (B | C | ...) & ...) + >>> to_cnf('~(B | C)') + (~B & ~C) + """ + s = expr(s) + if isinstance(s, str): + s = expr(s) + s = eliminate_implications(s) # Steps 1, 2 from p. 253 + s = move_not_inwards(s) # Step 3 + return distribute_and_over_or(s) # Step 4 + + +def eliminate_implications(s): + r"""Change implications into equivalent form with only &, \|, and ~ as logical operators.""" + s = expr(s) + if not s.args or is_symbol(s.op): + return s # Atoms are unchanged. + args = list(map(eliminate_implications, s.args)) + a, b = args[0], args[-1] + if s.op == '==>': + return b | ~a + elif s.op == '<==': + return a | ~b + elif s.op == '<=>': + return (a | ~b) & (b | ~a) + elif s.op == '^': + assert len(args) == 2 # TODO: relax this restriction + return (a & ~b) | (~a & b) + else: + assert s.op in ('&', '|', '~') + return Expr(s.op, *args) + + +def move_not_inwards(s): + """Rewrite sentence s by moving negation sign inward. + >>> move_not_inwards(~(A | B)) + (~A & ~B) + """ + s = expr(s) + if s.op == '~': + def NOT(b): + return move_not_inwards(~b) + + a = s.args[0] + if a.op == '~': + return move_not_inwards(a.args[0]) # ~~A ==> A + if a.op == '&': + return associate('|', list(map(NOT, a.args))) + if a.op == '|': + return associate('&', list(map(NOT, a.args))) + return s + elif is_symbol(s.op) or not s.args: + return s + else: + return Expr(s.op, *list(map(move_not_inwards, s.args))) + + +def distribute_and_over_or(s): + """Given a sentence s consisting of conjunctions and disjunctions + of literals, return an equivalent sentence in CNF. + >>> distribute_and_over_or((A & B) | C) + ((A | C) & (B | C)) + """ + s = expr(s) + if s.op == '|': + s = associate('|', s.args) + if s.op != '|': + return distribute_and_over_or(s) + if len(s.args) == 0: + return False + if len(s.args) == 1: + return distribute_and_over_or(s.args[0]) + conj = first(arg for arg in s.args if arg.op == '&') + if not conj: + return s + others = [a for a in s.args if a is not conj] + rest = associate('|', others) + return associate('&', [distribute_and_over_or(c | rest) + for c in conj.args]) + elif s.op == '&': + return associate('&', list(map(distribute_and_over_or, s.args))) + else: + return s + + +def associate(op, args): + r"""Given an associative op, return an expression with the same + meaning as Expr(op, \*args), but flattened -- that is, with nested + instances of the same op promoted to the top level. + + >>> associate('&', [(A&B),(B|C),(B&C)]) + (A & B & (B | C) & B & C) + >>> associate('|', [A|(B|(C|(A&B)))]) + (A | B | C | (A & B)) + """ + args = dissociate(op, args) + if len(args) == 0: + return _op_identity[op] + elif len(args) == 1: + return args[0] + else: + return Expr(op, *args) + + +_op_identity = {'&': True, '|': False, '+': 0, '*': 1} + + +def dissociate(op, args): + r"""Given an associative op, return a flattened list result such + that Expr(op, \*result) means the same as Expr(op, \*args). + + >>> dissociate('&', [A & B]) + [A, B] + """ + result = [] + + def collect(subargs): + for arg in subargs: + if arg.op == op: + collect(arg.args) + else: + result.append(arg) + + collect(args) + return result + + +def conjuncts(s): + """Return a list of the conjuncts in the sentence s. + >>> conjuncts(A & B) + [A, B] + >>> conjuncts(A | B) + [(A | B)] + """ + return dissociate('&', [s]) + + +def disjuncts(s): + """Return a list of the disjuncts in the sentence s. + >>> disjuncts(A | B) + [A, B] + >>> disjuncts(A & B) + [(A & B)] + """ + return dissociate('|', [s]) + + +# ______________________________________________________________________________ + + +def pl_resolution(kb, alpha): + """ + [Figure 7.12] + Propositional-logic resolution: say if alpha follows from KB. + >>> pl_resolution(horn_clauses_KB, A) + True + """ + clauses = kb.clauses + conjuncts(to_cnf(~alpha)) + new = set() + while True: + n = len(clauses) + pairs = [(clauses[i], clauses[j]) + for i in range(n) for j in range(i + 1, n)] + for (ci, cj) in pairs: + resolvents = pl_resolve(ci, cj) + if False in resolvents: + return True + new = new.union(set(resolvents)) + if new.issubset(set(clauses)): + return False + for c in new: + if c not in clauses: + clauses.append(c) + + +def pl_resolve(ci, cj): + """Return all clauses that can be obtained by resolving clauses ci and cj.""" + clauses = [] + for di in disjuncts(ci): + for dj in disjuncts(cj): + if di == ~dj or ~di == dj: + clauses.append(associate('|', unique(remove_all(di, disjuncts(ci)) + remove_all(dj, disjuncts(cj))))) + return clauses + + +# ______________________________________________________________________________ + + +class PropDefiniteKB(PropKB): + """A KB of propositional definite clauses.""" + + def tell(self, sentence): + """Add a definite clause to this KB.""" + assert is_definite_clause(sentence), "Must be definite clause" + self.clauses.append(sentence) + + def ask_generator(self, query): + """Yield the empty substitution if KB implies query; else nothing.""" + if pl_fc_entails(self.clauses, query): + yield {} + + def retract(self, sentence): + """Remove the given definite clause from this KB.""" + self.clauses.remove(sentence) + + def clauses_with_premise(self, p): + """Return a list of the clauses in KB that have p in their premise. + This could be cached away for O(1) speed, but we'll recompute it.""" + return [c for c in self.clauses if c.op == '==>' and p in conjuncts(c.args[0])] + + +def pl_fc_entails(kb, q): + """ + [Figure 7.15] + Use forward chaining to see if a PropDefiniteKB entails symbol q. + >>> pl_fc_entails(horn_clauses_KB, expr('Q')) + True + """ + count = {c: len(conjuncts(c.args[0])) for c in kb.clauses if c.op == '==>'} + inferred = defaultdict(bool) + agenda = [s for s in kb.clauses if is_prop_symbol(s.op)] + while agenda: + p = agenda.pop() + if p == q: + return True + if not inferred[p]: + inferred[p] = True + for c in kb.clauses_with_premise(p): + count[c] -= 1 + if count[c] == 0: + agenda.append(c.args[1]) + return False + + +""" +[Figure 7.13] +Simple inference in a wumpus world example +""" +wumpus_world_inference = expr('(B11 <=> (P12 | P21)) & ~B11') + +""" +[Figure 7.16] +Propositional Logic Forward Chaining example +""" +horn_clauses_KB = PropDefiniteKB() +for clause in ['P ==> Q', + '(L & M) ==> P', + '(B & L) ==> M', + '(A & P) ==> L', + '(A & B) ==> L', + 'A', 'B']: + horn_clauses_KB.tell(expr(clause)) + +""" +Definite clauses KB example +""" +definite_clauses_KB = PropDefiniteKB() +for clause in ['(B & F) ==> E', + '(A & E & F) ==> G', + '(B & C) ==> F', + '(A & B) ==> D', + '(E & F) ==> H', + '(H & I) ==>J', + 'A', 'B', 'C']: + definite_clauses_KB.tell(expr(clause)) + + +# ______________________________________________________________________________ +# Heuristics for SAT Solvers + + +def no_branching_heuristic(symbols, clauses): + """Default DPLL branching rule: pick the first remaining symbol and try True.""" + return first(symbols), True + + +def min_clauses(clauses): + """Return the clauses of minimum size (treating units as size 2 for MOMS-style heuristics).""" + min_len = min(map(lambda c: len(c.args), clauses), default=2) + return filter(lambda c: len(c.args) == (min_len if min_len > 1 else 2), clauses) + + +def moms(symbols, clauses): + """ + MOMS (Maximum Occurrence in clauses of Minimum Size) heuristic + Returns the literal with the most occurrences in all clauses of minimum size + """ + scores = Counter(l for c in min_clauses(clauses) for l in prop_symbols(c)) + return max(symbols, key=lambda symbol: scores[symbol]), True + + +def momsf(symbols, clauses, k=0): + """ + MOMS alternative heuristic + If f(x) the number of occurrences of the variable x in clauses with minimum size, + we choose the variable maximizing [f(x) + f(-x)] * 2^k + f(x) * f(-x) + Returns x if f(x) >= f(-x) otherwise -x + """ + scores = Counter(l for c in min_clauses(clauses) for l in disjuncts(c)) + P = max(symbols, + key=lambda symbol: (scores[symbol] + scores[~symbol]) * pow(2, k) + scores[symbol] * scores[~symbol]) + return P, True if scores[P] >= scores[~P] else False + + +def posit(symbols, clauses): + """ + Freeman's POSIT version of MOMs + Counts the positive x and negative x for each variable x in clauses with minimum size + Returns x if f(x) >= f(-x) otherwise -x + """ + scores = Counter(l for c in min_clauses(clauses) for l in disjuncts(c)) + P = max(symbols, key=lambda symbol: scores[symbol] + scores[~symbol]) + return P, True if scores[P] >= scores[~P] else False + + +def zm(symbols, clauses): + """ + Zabih and McAllester's version of MOMs + Counts the negative occurrences only of each variable x in clauses with minimum size + """ + scores = Counter(l for c in min_clauses(clauses) for l in disjuncts(c) if l.op == '~') + return max(symbols, key=lambda symbol: scores[~symbol]), True + + +def dlis(symbols, clauses): + """ + DLIS (Dynamic Largest Individual Sum) heuristic + Choose the variable and value that satisfies the maximum number of unsatisfied clauses + Like DLCS but we only consider the literal (thus Cp and Cn are individual) + """ + scores = Counter(l for c in clauses for l in disjuncts(c)) + P = max(symbols, key=lambda symbol: scores[symbol]) + return P, True if scores[P] >= scores[~P] else False + + +def dlcs(symbols, clauses): + """ + DLCS (Dynamic Largest Combined Sum) heuristic + Cp the number of clauses containing literal x + Cn the number of clauses containing literal -x + Here we select the variable maximizing Cp + Cn + Returns x if Cp >= Cn otherwise -x + """ + scores = Counter(l for c in clauses for l in disjuncts(c)) + P = max(symbols, key=lambda symbol: scores[symbol] + scores[~symbol]) + return P, True if scores[P] >= scores[~P] else False + + +def jw(symbols, clauses): + r""" + Jeroslow-Wang heuristic + For each literal compute J(l) = \sum{l in clause c} 2^{-\|c\|} + Return the literal maximizing J + """ + scores = Counter() + for c in clauses: + for l in prop_symbols(c): + scores[l] += pow(2, -len(c.args)) + return max(symbols, key=lambda symbol: scores[symbol]), True + + +def jw2(symbols, clauses): + """ + Two Sided Jeroslow-Wang heuristic + Compute J(l) also counts the negation of l = J(x) + J(-x) + Returns x if J(x) >= J(-x) otherwise -x + """ + scores = Counter() + for c in clauses: + for l in disjuncts(c): + scores[l] += pow(2, -len(c.args)) + P = max(symbols, key=lambda symbol: scores[symbol] + scores[~symbol]) + return P, True if scores[P] >= scores[~P] else False + + +# ______________________________________________________________________________ +# DPLL-Satisfiable [Figure 7.17] + + +def dpll_satisfiable(s, branching_heuristic=no_branching_heuristic): + """Check satisfiability of a propositional sentence. + This differs from the book code in two ways: (1) it returns a model + rather than True when it succeeds; this is more useful. (2) The + function find_pure_symbol is passed a list of unknown clauses, rather + than a list of all clauses and the model; this is more efficient. + + >>> dpll_satisfiable(A |'<=>'| B) == {A: True, B: True} + True + """ + return dpll(conjuncts(to_cnf(s)), prop_symbols(s), {}, branching_heuristic) + + +def dpll(clauses, symbols, model, branching_heuristic=no_branching_heuristic): + """[Figure 7.17] The Davis-Putnam-Logemann-Loveland procedure: decide whether + ``clauses`` are satisfiable by depth-first assignment of ``symbols``, accelerated + by the pure-symbol and unit-clause heuristics (plus an optional branching + heuristic). Returns the satisfying model, or False.""" + unknown_clauses = [] # clauses with an unknown truth value + for c in clauses: + val = pl_true(c, model) + if val is False: + return False + if val is None: + unknown_clauses.append(c) + if not unknown_clauses: + return model + P, value = find_pure_symbol(symbols, unknown_clauses) + if P: + return dpll(clauses, remove_all(P, symbols), extend(model, P, value), branching_heuristic) + P, value = find_unit_clause(clauses, model) + if P: + return dpll(clauses, remove_all(P, symbols), extend(model, P, value), branching_heuristic) + P, value = branching_heuristic(symbols, unknown_clauses) + return (dpll(clauses, remove_all(P, symbols), extend(model, P, value), branching_heuristic) or + dpll(clauses, remove_all(P, symbols), extend(model, P, not value), branching_heuristic)) + + +def find_pure_symbol(symbols, clauses): + """Find a symbol and its value if it appears only as a positive literal + (or only as a negative) in clauses. + >>> find_pure_symbol([A, B, C], [A|~B,~B|~C,C|A]) + (A, True) + """ + for s in symbols: + found_pos, found_neg = False, False + for c in clauses: + if not found_pos and s in disjuncts(c): + found_pos = True + if not found_neg and ~s in disjuncts(c): + found_neg = True + if found_pos != found_neg: + return s, found_pos + return None, None + + +def find_unit_clause(clauses, model): + """Find a forced assignment if possible from a clause with only 1 + variable not bound in the model. + >>> find_unit_clause([A|B|C, B|~C, ~A|~B], {A:True}) + (B, False) + """ + for clause in clauses: + P, value = unit_clause_assign(clause, model) + if P: + return P, value + return None, None + + +def unit_clause_assign(clause, model): + """Return a single variable/value pair that makes clause true in + the model, if possible. + >>> unit_clause_assign(A|B|C, {A:True}) + (None, None) + >>> unit_clause_assign(B|~C, {A:True}) + (None, None) + >>> unit_clause_assign(~A|~B, {A:True}) + (B, False) + """ + P, value = None, None + for literal in disjuncts(clause): + sym, positive = inspect_literal(literal) + if sym in model: + if model[sym] == positive: + return None, None # clause already True + elif P: + return None, None # more than 1 unbound variable + else: + P, value = sym, positive + return P, value + + +def inspect_literal(literal): + """The symbol in this literal, and the value it should take to + make the literal true. + >>> inspect_literal(P) + (P, True) + >>> inspect_literal(~P) + (P, False) + """ + if literal.op == '~': + return literal.args[0], False + else: + return literal, True + + +# ______________________________________________________________________________ +# CDCL - Conflict-Driven Clause Learning with 1UIP Learning Scheme, +# 2WL Lazy Data Structure, VSIDS Branching Heuristic & Restarts + + +def no_restart(conflicts, restarts, queue_lbd, sum_lbd): + """Default CDCL restart policy: never restart.""" + return False + + +def luby(conflicts, restarts, queue_lbd, sum_lbd, unit=512): + """Restart policy based on the Luby sequence (restart after ``unit * luby(restarts)`` conflicts).""" + # in the state-of-art tested with unit value 1, 2, 4, 6, 8, 12, 16, 32, 64, 128, 256 and 512 + def _luby(i): + k = 1 + while True: + if i == (1 << k) - 1: + return 1 << (k - 1) + elif (1 << (k - 1)) <= i < (1 << k) - 1: + return _luby(i - (1 << (k - 1)) + 1) + k += 1 + + return unit * _luby(restarts) == len(queue_lbd) + + +def glucose(conflicts, restarts, queue_lbd, sum_lbd, x=100, k=0.7): + """Glucose-style restart policy based on recent vs. global average literal block distance (LBD).""" + # in the state-of-art tested with (x, k) as (50, 0.8) and (100, 0.7) + # if there were at least x conflicts since the last restart, and then the average LBD of the last + # x learnt clauses was at least k times higher than the average LBD of all learnt clauses + return len(queue_lbd) >= x and sum(queue_lbd) / len(queue_lbd) * k > sum_lbd / conflicts + + +def cdcl_satisfiable(s, vsids_decay=0.95, restart_strategy=no_restart): + """ + >>> cdcl_satisfiable(A |'<=>'| B) == {A: True, B: True} + True + """ + clauses = TwoWLClauseDatabase(conjuncts(to_cnf(s))) + symbols = prop_symbols(s) + scores = Counter() + G = nx.DiGraph() + model = {} + dl = 0 + conflicts = 0 + restarts = 1 + sum_lbd = 0 + queue_lbd = [] + while True: + conflict = unit_propagation(clauses, symbols, model, G, dl) + if conflict: + if dl == 0: + return False + conflicts += 1 + dl, learn, lbd = conflict_analysis(G, dl) + queue_lbd.append(lbd) + sum_lbd += lbd + backjump(symbols, model, G, dl) + clauses.add(learn, model) + scores.update(l for l in disjuncts(learn)) + for symbol in scores: + scores[symbol] *= vsids_decay + if restart_strategy(conflicts, restarts, queue_lbd, sum_lbd): + backjump(symbols, model, G) + queue_lbd.clear() + restarts += 1 + else: + if not symbols: + return model + dl += 1 + assign_decision_literal(symbols, model, scores, G, dl) + + +def assign_decision_literal(symbols, model, scores, G, dl): + """Pick the unassigned symbol with the highest VSIDS score, assign it at decision level ``dl``, + and record it as a decision node in the implication graph ``G``.""" + P = max(symbols, key=lambda symbol: scores[symbol] + scores[~symbol]) + value = True if scores[P] >= scores[~P] else False + symbols.remove(P) + model[P] = value + G.add_node(P, val=value, dl=dl) + + +def unit_propagation(clauses, symbols, model, G, dl): + """Boolean constraint propagation over the two-watched-literal database. + + Repeatedly assigns forced literals from unit clauses (extending ``model`` and the implication + graph ``G``) until a fixpoint is reached. Returns True if a conflict clause is found, else False. + """ + + def check(c): + if not model or clauses.get_first_watched(c) == clauses.get_second_watched(c): + return True + w1, _ = inspect_literal(clauses.get_first_watched(c)) + if w1 in model: + return c in (clauses.get_neg_watched(w1) if model[w1] else clauses.get_pos_watched(w1)) + w2, _ = inspect_literal(clauses.get_second_watched(c)) + if w2 in model: + return c in (clauses.get_neg_watched(w2) if model[w2] else clauses.get_pos_watched(w2)) + + def unit_clause(watching): + w, p = inspect_literal(watching) + G.add_node(w, val=p, dl=dl) + G.add_edges_from(zip(prop_symbols(c) - {w}, itertools.cycle([w])), antecedent=c) + symbols.remove(w) + model[w] = p + + def conflict_clause(c): + G.add_edges_from(zip(prop_symbols(c), itertools.cycle('K')), antecedent=c) + + while True: + bcp = False + for c in filter(check, clauses.get_clauses()): + # we need only visit each clause when one of its two watched literals is assigned to 0 because, until + # this happens, we can guarantee that there cannot be more than n-2 literals in the clause assigned to 0 + first_watched = pl_true(clauses.get_first_watched(c), model) + second_watched = pl_true(clauses.get_second_watched(c), model) + if first_watched is None and clauses.get_first_watched(c) == clauses.get_second_watched(c): + unit_clause(clauses.get_first_watched(c)) + bcp = True + break + elif first_watched is False and second_watched is not True: + if clauses.update_second_watched(c, model): + bcp = True + else: + # if the only literal with a non-zero value is the other watched literal then + if second_watched is None: # if it is free, then the clause is a unit clause + unit_clause(clauses.get_second_watched(c)) + bcp = True + break + else: # else (it is False) the clause is a conflict clause + conflict_clause(c) + return True + elif second_watched is False and first_watched is not True: + if clauses.update_first_watched(c, model): + bcp = True + else: + # if the only literal with a non-zero value is the other watched literal then + if first_watched is None: # if it is free, then the clause is a unit clause + unit_clause(clauses.get_first_watched(c)) + bcp = True + break + else: # else (it is False) the clause is a conflict clause + conflict_clause(c) + return True + if not bcp: + return False + + +def conflict_analysis(G, dl): + """Analyse a conflict in the implication graph ``G`` using the first-UIP scheme. + + Resolves the conflict clause back to the first unique implication point and returns a tuple + ``(backjump_level, learnt_clause, lbd)`` where ``lbd`` is the learnt clause's literal block + distance. + """ + conflict_clause = next(G[p]['K']['antecedent'] for p in G.pred['K']) + P = next(node for node in G.nodes() - 'K' if G.nodes[node]['dl'] == dl and G.in_degree(node) == 0) + first_uip = nx.immediate_dominators(G, P)['K'] + G.remove_node('K') + conflict_side = nx.descendants(G, first_uip) + while True: + for l in prop_symbols(conflict_clause).intersection(conflict_side): + antecedent = next(G[p][l]['antecedent'] for p in G.pred[l]) + conflict_clause = pl_binary_resolution(conflict_clause, antecedent) + # the literal block distance is calculated by taking the decision levels from variables of all + # literals in the clause, and counting how many different decision levels were in this set + lbd = [G.nodes[l]['dl'] for l in prop_symbols(conflict_clause)] + if lbd.count(dl) == 1 and first_uip in prop_symbols(conflict_clause): + return 0 if len(lbd) == 1 else heapq.nlargest(2, lbd)[-1], conflict_clause, len(set(lbd)) + + +def pl_binary_resolution(ci, cj): + """Resolve two clauses on a complementary pair of literals and return the resolvent.""" + for di in disjuncts(ci): + for dj in disjuncts(cj): + if di == ~dj or ~di == dj: + return pl_binary_resolution(associate('|', remove_all(di, disjuncts(ci))), + associate('|', remove_all(dj, disjuncts(cj)))) + return associate('|', unique(disjuncts(ci) + disjuncts(cj))) + + +def backjump(symbols, model, G, dl=0): + """Undo all assignments made above decision level ``dl``, restoring those symbols as unassigned.""" + delete = {node for node in G.nodes() if G.nodes[node]['dl'] > dl} + G.remove_nodes_from(delete) + for node in delete: + del model[node] + symbols |= delete + + +class TwoWLClauseDatabase: + """Clause database using the two-watched-literal lazy data structure for fast unit propagation. + + Maintains, for each clause, two watched literals and, for each literal, the set of clauses in + which it is watched (split into positive and negative occurrences). + """ + + def __init__(self, clauses): + self.__twl = {} + self.__watch_list = defaultdict(lambda: [set(), set()]) + for c in clauses: + self.add(c, None) + + def get_clauses(self): + """Return a view of all clauses currently stored in the database.""" + return self.__twl.keys() + + def set_first_watched(self, clause, new_watching): + """Set the first watched literal of ``clause`` (no-op for clauses of length <= 2).""" + if len(clause.args) > 2: + self.__twl[clause][0] = new_watching + + def set_second_watched(self, clause, new_watching): + """Set the second watched literal of ``clause`` (no-op for clauses of length <= 2).""" + if len(clause.args) > 2: + self.__twl[clause][1] = new_watching + + def get_first_watched(self, clause): + """Return the first watched literal of ``clause``.""" + if len(clause.args) == 2: + return clause.args[0] + if len(clause.args) > 2: + return self.__twl[clause][0] + return clause + + def get_second_watched(self, clause): + """Return the second watched literal of ``clause``.""" + if len(clause.args) == 2: + return clause.args[-1] + if len(clause.args) > 2: + return self.__twl[clause][1] + return clause + + def get_pos_watched(self, l): + """Return the set of clauses in which symbol ``l`` is watched positively.""" + return self.__watch_list[l][0] + + def get_neg_watched(self, l): + """Return the set of clauses in which symbol ``l`` is watched negatively.""" + return self.__watch_list[l][1] + + def add(self, clause, model): + """Add ``clause`` to the database, choosing its two watched literals given ``model``.""" + self.__twl[clause] = self.__assign_watching_literals(clause, model) + w1, p1 = inspect_literal(self.get_first_watched(clause)) + w2, p2 = inspect_literal(self.get_second_watched(clause)) + self.__watch_list[w1][0].add(clause) if p1 else self.__watch_list[w1][1].add(clause) + if w1 != w2: + self.__watch_list[w2][0].add(clause) if p2 else self.__watch_list[w2][1].add(clause) + + def remove(self, clause): + """Remove ``clause`` from the database and from its watched literals' watch lists.""" + w1, p1 = inspect_literal(self.get_first_watched(clause)) + w2, p2 = inspect_literal(self.get_second_watched(clause)) + del self.__twl[clause] + self.__watch_list[w1][0].discard(clause) if p1 else self.__watch_list[w1][1].discard(clause) + if w1 != w2: + self.__watch_list[w2][0].discard(clause) if p2 else self.__watch_list[w2][1].discard(clause) + + def update_first_watched(self, clause, model): + """Try to replace the first watched literal of ``clause`` with another non-falsified literal. + + Returns True if a new watch was installed, otherwise None (the clause stays as is). + """ + # if a non-zero literal different from the other watched literal is found + found, new_watching = self.__find_new_watching_literal(clause, self.get_first_watched(clause), model) + if found: # then it will replace the watched literal + w, p = inspect_literal(self.get_second_watched(clause)) + self.__watch_list[w][0].remove(clause) if p else self.__watch_list[w][1].remove(clause) + self.set_second_watched(clause, new_watching) + w, p = inspect_literal(new_watching) + self.__watch_list[w][0].add(clause) if p else self.__watch_list[w][1].add(clause) + return True + + def update_second_watched(self, clause, model): + """Try to replace the second watched literal of ``clause`` with another non-falsified literal. + + Returns True if a new watch was installed, otherwise None (the clause stays as is). + """ + # if a non-zero literal different from the other watched literal is found + found, new_watching = self.__find_new_watching_literal(clause, self.get_second_watched(clause), model) + if found: # then it will replace the watched literal + w, p = inspect_literal(self.get_first_watched(clause)) + self.__watch_list[w][0].remove(clause) if p else self.__watch_list[w][1].remove(clause) + self.set_first_watched(clause, new_watching) + w, p = inspect_literal(new_watching) + self.__watch_list[w][0].add(clause) if p else self.__watch_list[w][1].add(clause) + return True + + def __find_new_watching_literal(self, clause, other_watched, model): + # if a non-zero literal different from the other watched literal is found + if len(clause.args) > 2: + for l in disjuncts(clause): + if l != other_watched and pl_true(l, model) is not False: + # then it is returned + return True, l + return False, None + + def __assign_watching_literals(self, clause, model=None): + if len(clause.args) > 2: + if model is None or not model: + return [clause.args[0], clause.args[-1]] + else: + return [next(l for l in disjuncts(clause) if pl_true(l, model) is None), + next(l for l in disjuncts(clause) if pl_true(l, model) is False)] + + +# ______________________________________________________________________________ +# Walk-SAT [Figure 7.18] + + +def WalkSAT(clauses, p=0.5, max_flips=10000): + """Checks for satisfiability of all clauses by randomly flipping values of variables + >>> WalkSAT([A & ~A], 0.5, 100) is None + True + """ + # Set of all symbols in all clauses + symbols = {sym for clause in clauses for sym in prop_symbols(clause)} + # model is a random assignment of true/false to the symbols in clauses + model = {s: random.choice([True, False]) for s in symbols} + for i in range(max_flips): + satisfied, unsatisfied = [], [] + for clause in clauses: + (satisfied if pl_true(clause, model) else unsatisfied).append(clause) + if not unsatisfied: # if model satisfies all the clauses + return model + clause = random.choice(unsatisfied) + if probability(p): + sym = random.choice(list(prop_symbols(clause))) + else: + # Flip the symbol in clause that maximizes number of sat. clauses + def sat_count(sym): + # Return the the number of clauses satisfied after flipping the symbol. + model[sym] = not model[sym] + count = len([clause for clause in clauses if pl_true(clause, model)]) + model[sym] = not model[sym] + return count + + sym = max(prop_symbols(clause), key=sat_count) + model[sym] = not model[sym] + # If no solution is found within the flip limit, we return failure + return None + + +# ______________________________________________________________________________ +# Map Coloring SAT Problems + + +def MapColoringSAT(colors, neighbors): + """Make a SAT for the problem of coloring a map with different colors + for any two adjacent regions. Arguments are a list of colors, and a + dict of {region: [neighbor,...]} entries. This dict may also be + specified as a string of the form defined by parse_neighbors.""" + if isinstance(neighbors, str): + neighbors = parse_neighbors(neighbors) + colors = UniversalDict(colors) + clauses = [] + for state in neighbors.keys(): + clause = [expr(state + '_' + c) for c in colors[state]] + clauses.append(clause) + for t in itertools.combinations(clause, 2): + clauses.append([~t[0], ~t[1]]) + visited = set() + adj = set(neighbors[state]) - visited + visited.add(state) + for n_state in adj: + for col in colors[n_state]: + clauses.append([expr('~' + state + '_' + col), expr('~' + n_state + '_' + col)]) + return associate('&', map(lambda c: associate('|', c), clauses)) + + +australia_sat = MapColoringSAT(list('RGB'), """SA: WA NT Q NSW V; NT: WA Q; NSW: Q V; T: """) + +france_sat = MapColoringSAT(list('RGBY'), + """AL: LO FC; AQ: MP LI PC; AU: LI CE BO RA LR MP; BO: CE IF CA FC RA + AU; BR: NB PL; CA: IF PI LO FC BO; CE: PL NB NH IF BO AU LI PC; FC: BO + CA LO AL RA; IF: NH PI CA BO CE; LI: PC CE AU MP AQ; LO: CA AL FC; LR: + MP AU RA PA; MP: AQ LI AU LR; NB: NH CE PL BR; NH: PI IF CE NB; NO: + PI; PA: LR RA; PC: PL CE LI AQ; PI: NH NO CA IF; PL: BR NB CE PC; RA: + AU BO FC PA LR""") + +usa_sat = MapColoringSAT(list('RGBY'), + """WA: OR ID; OR: ID NV CA; CA: NV AZ; NV: ID UT AZ; ID: MT WY UT; + UT: WY CO AZ; MT: ND SD WY; WY: SD NE CO; CO: NE KA OK NM; NM: OK TX AZ; + ND: MN SD; SD: MN IA NE; NE: IA MO KA; KA: MO OK; OK: MO AR TX; + TX: AR LA; MN: WI IA; IA: WI IL MO; MO: IL KY TN AR; AR: MS TN LA; + LA: MS; WI: MI IL; IL: IN KY; IN: OH KY; MS: TN AL; AL: TN GA FL; + MI: OH IN; OH: PA WV KY; KY: WV VA TN; TN: VA NC GA; GA: NC SC FL; + PA: NY NJ DE MD WV; WV: MD VA; VA: MD DC NC; NC: SC; NY: VT MA CT NJ; + NJ: DE; DE: MD; MD: DC; VT: NH MA; MA: NH RI CT; CT: RI; ME: NH; + HI: ; AK: """) + + +# ______________________________________________________________________________ + + +# Expr functions for WumpusKB and HybridWumpusAgent + +def facing_east(time): + """Proposition: the agent is facing east at the given time step.""" + return Expr('FacingEast', time) + + +def facing_west(time): + """Proposition: the agent is facing west at the given time step.""" + return Expr('FacingWest', time) + + +def facing_north(time): + """Proposition: the agent is facing north at the given time step.""" + return Expr('FacingNorth', time) + + +def facing_south(time): + """Proposition: the agent is facing south at the given time step.""" + return Expr('FacingSouth', time) + + +def wumpus(x, y): + """Proposition: there is a wumpus in square (x, y).""" + return Expr('W', x, y) + + +def pit(x, y): + """Proposition: there is a pit in square (x, y).""" + return Expr('P', x, y) + + +def breeze(x, y): + """Proposition: there is a breeze in square (x, y).""" + return Expr('B', x, y) + + +def stench(x, y): + """Proposition: there is a stench in square (x, y).""" + return Expr('S', x, y) + + +def wumpus_alive(time): + """Proposition: the wumpus is alive at the given time step.""" + return Expr('WumpusAlive', time) + + +def have_arrow(time): + """Proposition: the agent still has its arrow at the given time step.""" + return Expr('HaveArrow', time) + + +def percept_stench(time): + """Proposition: the agent perceives a stench at the given time step.""" + return Expr('Stench', time) + + +def percept_breeze(time): + """Proposition: the agent perceives a breeze at the given time step.""" + return Expr('Breeze', time) + + +def percept_glitter(time): + """Proposition: the agent perceives glitter at the given time step.""" + return Expr('Glitter', time) + + +def percept_bump(time): + """Proposition: the agent perceives a bump at the given time step.""" + return Expr('Bump', time) + + +def percept_scream(time): + """Proposition: the agent perceives a scream at the given time step.""" + return Expr('Scream', time) + + +def move_forward(time): + """Action proposition: the agent moves forward at the given time step.""" + return Expr('Forward', time) + + +def shoot(time): + """Action proposition: the agent shoots its arrow at the given time step.""" + return Expr('Shoot', time) + + +def turn_left(time): + """Action proposition: the agent turns left at the given time step.""" + return Expr('TurnLeft', time) + + +def turn_right(time): + """Action proposition: the agent turns right at the given time step.""" + return Expr('TurnRight', time) + + +def ok_to_move(x, y, time): + """Proposition: square (x, y) is safe to move into at the given time step.""" + return Expr('OK', x, y, time) + + +def location(x, y, time=None): + """Proposition for the agent being at square (x, y), optionally at a given time step.""" + if time is None: + return Expr('L', x, y) + else: + return Expr('L', x, y, time) + + +# Symbols + +def implies(lhs, rhs): + """Build the implication ``lhs ==> rhs`` as an Expr.""" + return Expr('==>', lhs, rhs) + + +def equiv(lhs, rhs): + """Build the biconditional ``lhs <=> rhs`` as an Expr.""" + return Expr('<=>', lhs, rhs) + + +# Helper Function + +def new_disjunction(sentences): + """Combine a list of sentences into a single disjunction (their logical OR).""" + t = sentences[0] + for i in range(1, len(sentences)): + t |= sentences[i] + return t + + +# ______________________________________________________________________________ + + +class WumpusKB(PropKB): + """ + Create a Knowledge Base that contains the a temporal "Wumpus physics" and temporal rules with time zero. + """ + + def __init__(self, dimrow): + super().__init__() + self.dimrow = dimrow + self.tell(~wumpus(1, 1)) + self.tell(~pit(1, 1)) + + for y in range(1, dimrow + 1): + for x in range(1, dimrow + 1): + + pits_in = list() + wumpus_in = list() + + if x > 1: # West room exists + pits_in.append(pit(x - 1, y)) + wumpus_in.append(wumpus(x - 1, y)) + + if y < dimrow: # North room exists + pits_in.append(pit(x, y + 1)) + wumpus_in.append(wumpus(x, y + 1)) + + if x < dimrow: # East room exists + pits_in.append(pit(x + 1, y)) + wumpus_in.append(wumpus(x + 1, y)) + + if y > 1: # South room exists + pits_in.append(pit(x, y - 1)) + wumpus_in.append(wumpus(x, y - 1)) + + self.tell(equiv(breeze(x, y), new_disjunction(pits_in))) + self.tell(equiv(stench(x, y), new_disjunction(wumpus_in))) + + # Rule that describes existence of at least one Wumpus + wumpus_at_least = list() + for x in range(1, dimrow + 1): + for y in range(1, dimrow + 1): + wumpus_at_least.append(wumpus(x, y)) + + self.tell(new_disjunction(wumpus_at_least)) + + # Rule that describes existence of at most one Wumpus + for i in range(1, dimrow + 1): + for j in range(1, dimrow + 1): + for u in range(1, dimrow + 1): + for v in range(1, dimrow + 1): + if i != u or j != v: + self.tell(~wumpus(i, j) | ~wumpus(u, v)) + + # Temporal rules at time zero + self.tell(location(1, 1, 0)) + for i in range(1, dimrow + 1): + for j in range(1, dimrow + 1): + self.tell(implies(location(i, j, 0), equiv(percept_breeze(0), breeze(i, j)))) + self.tell(implies(location(i, j, 0), equiv(percept_stench(0), stench(i, j)))) + # a square is safe to move into at time 0 iff it has no pit, no + # (live) wumpus -- without this the agent cannot even prove [1,1] + # safe on its first move (the rule was only added for time > 0) + self.tell(equiv(ok_to_move(i, j, 0), ~pit(i, j) & ~wumpus(i, j) & wumpus_alive(0))) + if i != 1 or j != 1: + self.tell(~location(i, j, 0)) + + self.tell(wumpus_alive(0)) + self.tell(have_arrow(0)) + self.tell(facing_east(0)) + self.tell(~facing_north(0)) + self.tell(~facing_south(0)) + self.tell(~facing_west(0)) + + def make_action_sentence(self, action, time): + """Tell the KB that ``action`` was taken at ``time`` and that no other action was.""" + actions = [move_forward(time), shoot(time), turn_left(time), turn_right(time)] + + for a in actions: + if action is a: + self.tell(action) + else: + self.tell(~a) + + def make_percept_sentence(self, percept, time): + """Tell the KB which of the five percepts hold (and which do not) at ``time``. + + A single square can yield several percepts at once (e.g. a Stench *and* a + Breeze), so ``percept`` may be one percept object or a collection of them + (the format produced by ``WumpusEnvironment.percept``); ``None`` means + nothing was perceived. + """ + if percept is None: + percepts = [] + elif isinstance(percept, (list, tuple, set)): + percepts = list(percept) + else: + percepts = [percept] + + # for each of the five percepts, assert whether it holds at this time + for cls, sentence in ((Glitter, percept_glitter), (Bump, percept_bump), + (Stench, percept_stench), (Breeze, percept_breeze), + (Scream, percept_scream)): + if any(isinstance(p, cls) for p in percepts): + self.tell(sentence(time)) + else: + self.tell(~sentence(time)) + + def add_temporal_sentences(self, time): + """Add the successor-state axioms relating ``time`` to ``time - 1`` (location, orientation, + arrow possession and wumpus status). Does nothing for ``time == 0``.""" + if time == 0: + return + t = time - 1 + + # current location rules + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + self.tell(implies(location(i, j, time), equiv(percept_breeze(time), breeze(i, j)))) + self.tell(implies(location(i, j, time), equiv(percept_stench(time), stench(i, j)))) + s = list() + s.append(equiv(location(i, j, time), location(i, j, time) & ~move_forward(time) | percept_bump(time))) + if i != 1: + s.append(location(i - 1, j, t) & facing_east(t) & move_forward(t)) + if i != self.dimrow: + s.append(location(i + 1, j, t) & facing_west(t) & move_forward(t)) + if j != 1: + s.append(location(i, j - 1, t) & facing_north(t) & move_forward(t)) + if j != self.dimrow: + s.append(location(i, j + 1, t) & facing_south(t) & move_forward(t)) + + # add sentence about location i,j + self.tell(new_disjunction(s)) + + # add sentence about safety of location i,j + self.tell(equiv(ok_to_move(i, j, time), ~pit(i, j) & ~wumpus(i, j) & wumpus_alive(time))) + + # Rules about current orientation + + a = facing_north(t) & turn_right(t) + b = facing_south(t) & turn_left(t) + c = facing_east(t) & ~turn_left(t) & ~turn_right(t) + s = equiv(facing_east(time), a | b | c) + self.tell(s) + + a = facing_north(t) & turn_left(t) + b = facing_south(t) & turn_right(t) + c = facing_west(t) & ~turn_left(t) & ~turn_right(t) + s = equiv(facing_west(time), a | b | c) + self.tell(s) + + a = facing_east(t) & turn_left(t) + b = facing_west(t) & turn_right(t) + c = facing_north(t) & ~turn_left(t) & ~turn_right(t) + s = equiv(facing_north(time), a | b | c) + self.tell(s) + + a = facing_west(t) & turn_left(t) + b = facing_east(t) & turn_right(t) + c = facing_south(t) & ~turn_left(t) & ~turn_right(t) + s = equiv(facing_south(time), a | b | c) + self.tell(s) + + # Rules about last action: moving forward implies the agent did not turn + # (the converse does not hold -- the agent may instead grab, shoot or climb) + self.tell(implies(move_forward(t), ~turn_right(t) & ~turn_left(t))) + + # Rule about the arrow + self.tell(equiv(have_arrow(time), have_arrow(t) & ~shoot(t))) + + # Rule about Wumpus (dead or alive) + self.tell(equiv(wumpus_alive(time), wumpus_alive(t) & ~percept_scream(time))) + + def ask_if_true(self, query): + """Return True if the KB entails ``query``, decided via a SAT check on ``KB & ~query``.""" + # the KB entails the query iff KB & ~query is unsatisfiable; using a SAT + # solver here instead of pl_resolution keeps inference tractable on the + # large wumpus clause set (full resolution closure does not terminate) + return not dpll_satisfiable(associate('&', self.clauses + conjuncts(to_cnf(~query)))) + + +# ______________________________________________________________________________ + + +class WumpusPosition: + """A pose in the wumpus world: a square (x, y) together with an orientation.""" + + def __init__(self, x, y, orientation): + self.X = x + self.Y = y + self.orientation = orientation + + def get_location(self): + """Return the (x, y) coordinates of this position.""" + return self.X, self.Y + + def set_location(self, x, y): + """Set the (x, y) coordinates of this position.""" + self.X = x + self.Y = y + + def get_orientation(self): + """Return the orientation of this position.""" + return self.orientation + + def set_orientation(self, orientation): + """Set the orientation of this position.""" + self.orientation = orientation + + def __eq__(self, other): + if other.get_location() == self.get_location() and other.get_orientation() == self.get_orientation(): + return True + else: + return False + + def __hash__(self): + return hash((self.X, self.Y, self.orientation)) + + +# ______________________________________________________________________________ + + +class HybridWumpusAgent(Agent): + """ + [Figure 7.20] + An agent for the wumpus world that does logical inference. + """ + + def __init__(self, dimensions): + self.dimrow = dimensions + self.kb = WumpusKB(self.dimrow) + self.t = 0 + self.plan = list() + self.current_position = WumpusPosition(1, 1, 'UP') + super().__init__(self.execute) + + def execute(self, percept): + """Update the KB with ``percept``, infer the current state, and return the next action. + + Implements the agent program: it deduces the current pose and safe squares, then either + grabs the gold, explores unvisited safe squares, shoots at a possible wumpus, takes a + calculated risk, or heads home and climbs out. + """ + self.kb.make_percept_sentence(percept, self.t) + self.kb.add_temporal_sentences(self.t) + + temp = list() + + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + if self.kb.ask_if_true(location(i, j, self.t)): + temp.append(i) + temp.append(j) + + if len(temp) >= 2: # a location was entailed; otherwise keep the previous pose + if self.kb.ask_if_true(facing_north(self.t)): + self.current_position = WumpusPosition(temp[0], temp[1], 'UP') + elif self.kb.ask_if_true(facing_south(self.t)): + self.current_position = WumpusPosition(temp[0], temp[1], 'DOWN') + elif self.kb.ask_if_true(facing_west(self.t)): + self.current_position = WumpusPosition(temp[0], temp[1], 'LEFT') + elif self.kb.ask_if_true(facing_east(self.t)): + self.current_position = WumpusPosition(temp[0], temp[1], 'RIGHT') + + safe_points = list() + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + if self.kb.ask_if_true(ok_to_move(i, j, self.t)): + safe_points.append([i, j]) + + if self.kb.ask_if_true(percept_glitter(self.t)): + goals = list() + goals.append([1, 1]) + self.plan.append('Grab') + actions = self.plan_route(self.current_position, goals, safe_points) + self.plan.extend(actions) + self.plan.append('Climb') + + if len(self.plan) == 0: + # unvisited = squares the agent has never been located in (for all t' <= t) + unvisited = list() + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + if not any(self.kb.ask_if_true(location(i, j, k)) for k in range(self.t + 1)): + unvisited.append([i, j]) + unvisited_and_safe = [u for u in unvisited if u in safe_points] + + temp = self.plan_route(self.current_position, unvisited_and_safe, safe_points) + self.plan.extend(temp) + + if len(self.plan) == 0 and self.kb.ask_if_true(have_arrow(self.t)): + possible_wumpus = list() + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + if not self.kb.ask_if_true(~wumpus(i, j)): + possible_wumpus.append([i, j]) + + temp = self.plan_shot(self.current_position, possible_wumpus, safe_points) + self.plan.extend(temp) + + if len(self.plan) == 0: + not_unsafe = list() + for i in range(1, self.dimrow + 1): + for j in range(1, self.dimrow + 1): + if not self.kb.ask_if_true(~ok_to_move(i, j, self.t)): + not_unsafe.append([i, j]) + temp = self.plan_route(self.current_position, not_unsafe, safe_points) + self.plan.extend(temp) + + if len(self.plan) == 0: + start = list() + start.append([1, 1]) + temp = self.plan_route(self.current_position, start, safe_points) + self.plan.extend(temp) + self.plan.append('Climb') + + action = self.plan[0] + self.plan = self.plan[1:] + self.kb.make_action_sentence(action, self.t) + self.t += 1 + + return action + + def plan_route(self, current, goals, allowed): + """Return an action sequence from ``current`` to any of ``goals`` through ``allowed`` squares, + found by A* search.""" + problem = PlanRoute(current, goals, allowed, self.dimrow) + node = astar_search(problem) + return node.solution() if node is not None else [] + + def plan_shot(self, current, goals, allowed): + """Return an action sequence that moves to a square lined up with a possible wumpus and shoots.""" + shooting_positions = set() + + for loc in goals: + x = loc[0] + y = loc[1] + for i in range(1, self.dimrow + 1): + if i < x: + shooting_positions.add(WumpusPosition(i, y, 'RIGHT')) # face east toward the wumpus + if i > x: + shooting_positions.add(WumpusPosition(i, y, 'LEFT')) # face west + if i < y: + shooting_positions.add(WumpusPosition(x, i, 'UP')) # face north + if i > y: + shooting_positions.add(WumpusPosition(x, i, 'DOWN')) # face south + + # Can't have a shooting position from any of the rooms the Wumpus could reside + orientations = ['UP', 'DOWN', 'LEFT', 'RIGHT'] + for loc in goals: + for orientation in orientations: + shooting_positions.discard(WumpusPosition(loc[0], loc[1], orientation)) + + actions = list() + actions.extend(self.plan_route(current, shooting_positions, allowed)) + actions.append('Shoot') + return actions + + +# ______________________________________________________________________________ + + +def SAT_plan(init, transition, goal, t_max, SAT_solver=cdcl_satisfiable): + """ + [Figure 7.22] + Converts a planning problem to Satisfaction problem by translating it to a cnf sentence. + >>> transition = {'A': {'Left': 'A', 'Right': 'B'}, 'B': {'Left': 'A', 'Right': 'C'}, 'C': {'Left': 'B', 'Right': 'C'}} + >>> SAT_plan('A', transition, 'C', 1) is None + True + """ + + # Functions used by SAT_plan + def translate_to_SAT(init, transition, goal, time): + clauses = [] + states = [state for state in transition] + + # Symbol claiming state s at time t + state_counter = itertools.count() + for s in states: + for t in range(time + 1): + state_sym[s, t] = Expr('S_{}'.format(next(state_counter))) + + # Add initial state axiom + clauses.append(state_sym[init, 0]) + + # Add goal state axiom + clauses.append(state_sym[first(clause[0] for clause in state_sym + if set(conjuncts(clause[0])).issuperset(conjuncts(goal))), time]) \ + if isinstance(goal, Expr) else clauses.append(state_sym[goal, time]) + + # All possible transitions + transition_counter = itertools.count() + for s in states: + for action in transition[s]: + s_ = transition[s][action] + for t in range(time): + # Action 'action' taken from state 's' at time 't' to reach 's_' + action_sym[s, action, t] = Expr('T_{}'.format(next(transition_counter))) + + # Change the state from s to s_ + clauses.append(action_sym[s, action, t] | '==>' | state_sym[s, t]) + clauses.append(action_sym[s, action, t] | '==>' | state_sym[s_, t + 1]) + + # Allow only one state at any time + for t in range(time + 1): + # must be a state at any time + clauses.append(associate('|', [state_sym[s, t] for s in states])) + + for s in states: + for s_ in states[states.index(s) + 1:]: + # for each pair of states s, s_ only one is possible at time t + clauses.append((~state_sym[s, t]) | (~state_sym[s_, t])) + + # Restrict to one transition per timestep + for t in range(time): + # list of possible transitions at time t + transitions_t = [tr for tr in action_sym if tr[2] == t] + + # make sure at least one of the transitions happens + clauses.append(associate('|', [action_sym[tr] for tr in transitions_t])) + + for tr in transitions_t: + for tr_ in transitions_t[transitions_t.index(tr) + 1:]: + # there cannot be two transitions tr and tr_ at time t + clauses.append(~action_sym[tr] | ~action_sym[tr_]) + + # Combine the clauses to form the cnf + return associate('&', clauses) + + def extract_solution(model): + true_transitions = [t for t in action_sym if model[action_sym[t]]] + # Sort transitions based on time, which is the 3rd element of the tuple + true_transitions.sort(key=lambda x: x[2]) + return [action for s, action, time in true_transitions] + + # Body of SAT_plan algorithm + for t in range(t_max + 1): + # dictionaries to help extract the solution from model + state_sym = {} + action_sym = {} + + cnf = translate_to_SAT(init, transition, goal, t) + model = SAT_solver(cnf) + if model is not False: + return extract_solution(model) + return None + + +# ______________________________________________________________________________ + + +def unify(x, y, s={}): + """ + [Figure 9.1] + Unify expressions x,y with substitution s; return a substitution that + would make x,y equal, or None if x,y can not unify. x and y can be + variables (e.g. Expr('x')), constants, lists, or Exprs. + >>> unify(x, 3, {}) + {x: 3} + """ + if s is None: + return None + elif x == y: + return s + elif is_variable(x): + return unify_var(x, y, s) + elif is_variable(y): + return unify_var(y, x, s) + elif isinstance(x, Expr) and isinstance(y, Expr): + return unify(x.args, y.args, unify(x.op, y.op, s)) + elif isinstance(x, str) or isinstance(y, str): + return None + elif issequence(x) and issequence(y) and len(x) == len(y): + if not x: + return s + return unify(x[1:], y[1:], unify(x[0], y[0], s)) + else: + return None + + +def is_variable(x): + """A variable is an Expr with no args and a lowercase symbol as the op.""" + return isinstance(x, Expr) and not x.args and x.op[0].islower() + + +def unify_var(var, x, s): + """Unify the variable ``var`` with ``x`` under substitution ``s``. + + Follows existing bindings, applies the occur-check to avoid cyclic bindings, and returns the + extended substitution (or None if unification fails). + """ + if var in s: + return unify(s[var], x, s) + elif x in s: + return unify(var, s[x], s) + elif occur_check(var, x, s): + return None + else: + new_s = extend(s, var, x) + cascade_substitution(new_s) + return new_s + + +def occur_check(var, x, s): + """Return true if variable var occurs anywhere in x + (or in subst(s, x), if s has a binding for x).""" + if var == x: + return True + elif is_variable(x) and x in s: + return occur_check(var, s[x], s) + elif isinstance(x, Expr): + return (occur_check(var, x.op, s) or + occur_check(var, x.args, s)) + elif isinstance(x, (list, tuple)): + return any(occur_check(var, e, s) for e in x) + else: + return False + + +def subst(s, x): + """Substitute the substitution s into the expression x. + >>> subst({x: 42, y:0}, F(x) + y) + (F(42) + 0) + """ + if isinstance(x, list): + return [subst(s, xi) for xi in x] + elif isinstance(x, tuple): + return tuple([subst(s, xi) for xi in x]) + elif not isinstance(x, Expr): + return x + elif is_var_symbol(x.op): + return s.get(x, x) + else: + return Expr(x.op, *[subst(s, arg) for arg in x.args]) + + +def cascade_substitution(s): + """This method allows to return a correct unifier in normal form + and perform a cascade substitution to s. + For every mapping in s perform a cascade substitution on s.get(x) + and if it is replaced with a function ensure that all the function + terms are correct updates by passing over them again. + >>> s = {x: y, y: G(z)} + >>> cascade_substitution(s) + >>> s == {x: G(z), y: G(z)} + True + """ + + for x in s: + s[x] = subst(s, s.get(x)) + if isinstance(s.get(x), Expr) and not is_variable(s.get(x)): + # Ensure Function Terms are correct updates by passing over them again + s[x] = subst(s, s.get(x)) + + +def unify_mm(x, y, s={}): + """Unify expressions x,y with substitution s using an efficient rule-based + unification algorithm by Martelli & Montanari; return a substitution that + would make x,y equal, or None if x,y can not unify. x and y can be + variables (e.g. Expr('x')), constants, lists, or Exprs. + >>> unify_mm(x, 3, {}) + {x: 3} + """ + + set_eq = extend(s, x, y) + s = set_eq.copy() + while True: + trans = 0 + for x, y in set_eq.items(): + if x == y: + # if x = y this mapping is deleted (rule b) + del s[x] + elif not is_variable(x) and is_variable(y): + # if x is not a variable and y is a variable, rewrite it as y = x in s (rule a) + if s.get(y, None) is None: + s[y] = x + del s[x] + else: + # if a mapping already exist for variable y then apply + # variable elimination (there is a chance to apply rule d) + s[x] = vars_elimination(y, s) + elif not is_variable(x) and not is_variable(y): + # in which case x and y are not variables, if the two root function symbols + # are different, stop with failure, else apply term reduction (rule c) + if x.op is y.op and len(x.args) == len(y.args): + term_reduction(x, y, s) + del s[x] + else: + return None + elif isinstance(y, Expr): + # in which case x is a variable and y is a function or a variable (e.g. F(z) or y), + # if y is a function, we must check if x occurs in y, then stop with failure, else + # try to apply variable elimination to y (rule d) + if occur_check(x, y, s): + return None + s[x] = vars_elimination(y, s) + if y == s.get(x): + trans += 1 + else: + trans += 1 + if trans == len(set_eq): + # if no transformation has been applied, stop with success + return s + set_eq = s.copy() + + +def term_reduction(x, y, s): + """Apply term reduction to x and y if both are functions and the two root function + symbols are equals (e.g. F(x1, x2, ..., xn) and F(x1', x2', ..., xn')) by returning + a new mapping obtained by replacing x: y with {x1: x1', x2: x2', ..., xn: xn'} + """ + for i in range(len(x.args)): + if x.args[i] in s: + s[s.get(x.args[i])] = y.args[i] + else: + s[x.args[i]] = y.args[i] + + +def vars_elimination(x, s): + """Apply variable elimination to x: if x is a variable and occurs in s, return + the term mapped by x, else if x is a function recursively applies variable + elimination to each term of the function.""" + if not isinstance(x, Expr): + return x + if is_variable(x): + return s.get(x, x) + return Expr(x.op, *[vars_elimination(arg, s) for arg in x.args]) + + +def standardize_variables(sentence, dic=None): + """Replace all the variables in sentence with new variables.""" + if dic is None: + dic = {} + if not isinstance(sentence, Expr): + return sentence + elif is_var_symbol(sentence.op): + if sentence in dic: + return dic[sentence] + else: + v = Expr('v_{}'.format(next(standardize_variables.counter))) + dic[sentence] = v + return v + else: + return Expr(sentence.op, *[standardize_variables(a, dic) for a in sentence.args]) + + +standardize_variables.counter = itertools.count() + + +# ______________________________________________________________________________ + + +def parse_clauses_from_dimacs(dimacs_cnf): + """Converts a string into CNF clauses according to the DIMACS format used in SAT competitions""" + return map(lambda c: associate('|', c), + map(lambda c: [expr('~X' + str(abs(l))) if l < 0 else expr('X' + str(l)) for l in c], + map(lambda line: map(int, line.split()), + filter(None, ' '.join( + filter(lambda line: line[0] not in ('c', 'p'), + filter(None, dimacs_cnf.strip().replace('\t', ' ').split('\n')))).split(' 0'))))) + + +# ______________________________________________________________________________ + + +class FolKB(KB): + """A knowledge base consisting of first-order definite clauses. + >>> kb0 = FolKB([expr('Farmer(Mac)'), expr('Rabbit(Pete)'), + ... expr('(Rabbit(r) & Farmer(f)) ==> Hates(f, r)')]) + >>> kb0.tell(expr('Rabbit(Flopsie)')) + >>> kb0.retract(expr('Rabbit(Pete)')) + >>> kb0.ask(expr('Hates(Mac, x)'))[x] + Flopsie + >>> kb0.ask(expr('Wife(Pete, x)')) + False + """ + + def __init__(self, clauses=None): + super().__init__() + self.clauses = [] # inefficient: no indexing + if clauses: + for clause in clauses: + self.tell(clause) + + def tell(self, sentence): + """Add a definite clause to the KB, raising an exception if it is not one.""" + if is_definite_clause(sentence): + self.clauses.append(sentence) + else: + raise Exception('Not a definite clause: {}'.format(sentence)) + + def ask_generator(self, query): + """Yield substitutions under which the KB entails ``query`` (via backward chaining).""" + return fol_bc_ask(self, query) + + def retract(self, sentence): + """Remove the given clause from the KB.""" + self.clauses.remove(sentence) + + def fetch_rules_for_goal(self, goal): + """Return the clauses that could be used to prove ``goal`` (here, all clauses).""" + return self.clauses + + +def fol_fc_ask(kb, alpha): + """ + [Figure 9.3] + A simple forward-chaining algorithm. + """ + # TODO: improve efficiency + kb_consts = list({c for clause in kb.clauses for c in constant_symbols(clause)}) + + def enum_subst(p): + query_vars = list({v for clause in p for v in variables(clause)}) + for assignment_list in itertools.product(kb_consts, repeat=len(query_vars)): + theta = {x: y for x, y in zip(query_vars, assignment_list)} + yield theta + + # check if we can answer without new inferences + for q in kb.clauses: + phi = unify_mm(q, alpha) + if phi is not None: + yield phi + + while True: + new = [] + for rule in kb.clauses: + p, q = parse_definite_clause(rule) + for theta in enum_subst(p): + if set(subst(theta, p)).issubset(set(kb.clauses)): + q_ = subst(theta, q) + if all([unify_mm(x, q_) is None for x in kb.clauses + new]): + new.append(q_) + phi = unify_mm(q_, alpha) + if phi is not None: + yield phi + if not new: + break + for clause in new: + kb.tell(clause) + return None + + +def fol_bc_ask(kb, query): + """ + [Figure 9.6] + A simple backward-chaining algorithm for first-order logic. + KB should be an instance of FolKB, and query an atomic sentence. + """ + return fol_bc_or(kb, query, {}) + + +def fol_bc_or(kb, goal, theta): + """Backward-chaining OR step: yield substitutions proving ``goal`` via some rule in ``kb``. + + See [Figure 9.6]. + """ + for rule in kb.fetch_rules_for_goal(goal): + lhs, rhs = parse_definite_clause(standardize_variables(rule)) + for theta1 in fol_bc_and(kb, lhs, unify_mm(rhs, goal, theta)): + yield theta1 + + +def fol_bc_and(kb, goals, theta): + """Backward-chaining AND step: yield substitutions proving every goal in ``goals``. + + See [Figure 9.6]. + """ + if theta is None: + pass + elif not goals: + yield theta + else: + first, rest = goals[0], goals[1:] + for theta1 in fol_bc_or(kb, subst(theta, first), theta): + for theta2 in fol_bc_and(kb, rest, theta1): + yield theta2 + + +# A simple KB that defines the relevant conditions of the Wumpus World as in Figure 7.4. +# See Sec. 7.4.3 +wumpus_kb = PropKB() + +P11, P12, P21, P22, P31, B11, B21 = expr('P11, P12, P21, P22, P31, B11, B21') +wumpus_kb.tell(~P11) +wumpus_kb.tell(B11 | '<=>' | (P12 | P21)) +wumpus_kb.tell(B21 | '<=>' | (P11 | P22 | P31)) +wumpus_kb.tell(~B11) +wumpus_kb.tell(B21) + +test_kb = FolKB(map(expr, ['Farmer(Mac)', + 'Rabbit(Pete)', + 'Mother(MrsMac, Mac)', + 'Mother(MrsRabbit, Pete)', + '(Rabbit(r) & Farmer(f)) ==> Hates(f, r)', + '(Mother(m, c)) ==> Loves(m, c)', + '(Mother(m, r) & Rabbit(r)) ==> Rabbit(m)', + '(Farmer(f)) ==> Human(f)', + # Note that this order of conjuncts + # would result in infinite recursion: + # '(Human(h) & Mother(m, h)) ==> Human(m)' + '(Mother(m, h) & Human(h)) ==> Human(m)'])) + +crime_kb = FolKB(map(expr, ['(American(x) & Weapon(y) & Sells(x, y, z) & Hostile(z)) ==> Criminal(x)', + 'Owns(Nono, M1)', + 'Missile(M1)', + '(Missile(x) & Owns(Nono, x)) ==> Sells(West, x, Nono)', + 'Missile(x) ==> Weapon(x)', + 'Enemy(x, America) ==> Hostile(x)', + 'American(West)', + 'Enemy(Nono, America)'])) + + +# ______________________________________________________________________________ + +# Example application (not in the book). +# You can use the Expr class to do symbolic differentiation. This used to be +# a part of AI; now it is considered a separate field, Symbolic Algebra. + + +def diff(y, x): + """Return the symbolic derivative, dy/dx, as an Expr. + However, you probably want to simplify the results with simp. + >>> diff(x * x, x) + ((x * 1) + (x * 1)) + """ + if y == x: + return 1 + elif not y.args: + return 0 + else: + u, op, v = y.args[0], y.op, y.args[-1] + if op == '+': + return diff(u, x) + diff(v, x) + elif op == '-' and len(y.args) == 1: + return -diff(u, x) + elif op == '-': + return diff(u, x) - diff(v, x) + elif op == '*': + return u * diff(v, x) + v * diff(u, x) + elif op == '/': + return (v * diff(u, x) - u * diff(v, x)) / (v * v) + elif op == '**' and isnumber(x.op): + return v * u ** (v - 1) * diff(u, x) + elif op == '**': + return (v * u ** (v - 1) * diff(u, x) + + u ** v * Expr('log')(u) * diff(v, x)) + elif op == 'log': + return diff(u, x) / u + else: + raise ValueError('Unknown op: {} in diff({}, {})'.format(op, y, x)) + + +def simp(x): + """Simplify the expression x.""" + if isnumber(x) or not x.args: + return x + args = list(map(simp, x.args)) + u, op, v = args[0], x.op, args[-1] + if op == '+': + if v == 0: + return u + if u == 0: + return v + if u == v: + return 2 * u + if u == -v or v == -u: + return 0 + elif op == '-' and len(args) == 1: + if u.op == '-' and len(u.args) == 1: + return u.args[0] # --y ==> y + elif op == '-': + if v == 0: + return u + if u == 0: + return -v + if u == v: + return 0 + if u == -v or v == -u: + return 0 + elif op == '*': + if u == 0 or v == 0: + return 0 + if u == 1: + return v + if v == 1: + return u + if u == v: + return u ** 2 + elif op == '/': + if u == 0: + return 0 + if v == 0: + return Expr('Undefined') + if u == v: + return 1 + if u == -v or v == -u: + return 0 + elif op == '**': + if u == 0: + return 0 + if v == 0: + return 1 + if u == 1: + return 1 + if v == 1: + return u + elif op == 'log': + if u == 1: + return 0 + else: + raise ValueError('Unknown op: ' + op) + # If we fall through to here, we can not simplify further + return Expr(op, *args) + + +def d(y, x): + """Differentiate and then simplify. + >>> d(x * x - x, x) + ((2 * x) - 1) + """ + return simp(diff(y, x)) diff --git a/q-learning-9b/aima/making_simple_decisions.py b/q-learning-9b/aima/making_simple_decisions.py new file mode 100644 index 0000000..9a28037 --- /dev/null +++ b/q-learning-9b/aima/making_simple_decisions.py @@ -0,0 +1,11 @@ +"""Making Simple Decisions (Chapter 15/16) + +The decision-theoretic algorithms — decision networks and the +information-gathering agent — are implemented in :mod:`aima.probability`, +alongside the Bayesian-network machinery they build on. They are re-exported +here under the chapter's name; :mod:`aima.probability` is their canonical home. +""" + +from aima.probability import DecisionNetwork, InformationGatheringAgent + +__all__ = ['DecisionNetwork', 'InformationGatheringAgent'] diff --git a/q-learning-9b/aima/mdp.py b/q-learning-9b/aima/mdp.py new file mode 100644 index 0000000..0884eb8 --- /dev/null +++ b/q-learning-9b/aima/mdp.py @@ -0,0 +1,593 @@ +""" +Markov Decision Processes (Chapter 17) + +First we define an MDP, and the special case of a GridMDP, in which +states are laid out in a 2-dimensional grid. We also represent a policy +as a dictionary of {state: action} pairs, and a Utility function as a +dictionary of {state: number} pairs. We then define the value_iteration +and policy_iteration algorithms. +""" + +import random +from collections import defaultdict + +import numpy as np + +from aima.utils import vector_add, orientations, turn_right, turn_left + + +class MDP: + """A Markov Decision Process, defined by an initial state, transition model, + and reward function. We also keep track of a gamma value, for use by + algorithms. The transition model is represented somewhat differently from + the text. Instead of P(s' | s, a) being a probability number for each + state/state/action triplet, we instead have T(s, a) return a + list of (p, s') pairs. We also keep track of the possible states, + terminal states, and actions for each state. [Page 646]""" + + def __init__(self, init, actlist, terminals, transitions=None, reward=None, states=None, gamma=0.9): + if not (0 < gamma <= 1): + raise ValueError("An MDP must have 0 < gamma <= 1") + + # collect states from transitions table if not passed. + self.states = states or self.get_states_from_transitions(transitions) + + self.init = init + + if isinstance(actlist, list): + # if actlist is a list, all states have the same actions + self.actlist = actlist + + elif isinstance(actlist, dict): + # if actlist is a dict, different actions for each state + self.actlist = actlist + + elif isinstance(actlist, set): + # if actlist is a set, convert it to a list so actions are indexable + self.actlist = list(actlist) + + self.terminals = terminals + self.transitions = transitions or {} + if not self.transitions: + print("Warning: Transition table is empty.") + + self.gamma = gamma + + self.reward = reward or {s: 0 for s in self.states} + + # self.check_consistency() + + def R(self, state): + """Return a numeric reward for this state.""" + + return self.reward[state] + + def T(self, state, action): + """Transition model. From a state and an action, return a list + of (probability, result-state) pairs.""" + + if not self.transitions: + raise ValueError("Transition model is missing") + else: + return self.transitions[state][action] + + def actions(self, state): + """Return a list of actions that can be performed in this state. By default, a + fixed list of actions, except for terminal states. Override this + method if you need to specialize by state.""" + + if state in self.terminals: + return [None] + else: + return self.actlist + + def get_states_from_transitions(self, transitions): + """Return the set of all states mentioned in the transition model, both as source + states and as reachable next states (or None if transitions is not a dict).""" + if isinstance(transitions, dict): + s1 = set(transitions.keys()) + s2 = set(tr[1] for actions in transitions.values() + for effects in actions.values() + for tr in effects) + return s1.union(s2) + else: + print('Could not retrieve states from transitions') + return None + + def check_consistency(self): + """Assert that the MDP is well formed: the states match those in the transitions, + the initial and terminal states are valid, rewards are defined for every state, and + each action's outcome probabilities sum to 1.""" + + # check that all states in transitions are valid + assert set(self.states) == self.get_states_from_transitions(self.transitions) + + # check that init is a valid state + assert self.init in self.states + + # check reward for each state + assert set(self.reward.keys()) == set(self.states) + + # check that all terminals are valid states + assert all(t in self.states for t in self.terminals) + + # check that probability distributions for all actions sum to 1 + for s1, actions in self.transitions.items(): + for a in actions.keys(): + s = 0 + for o in actions[a]: + s += o[0] + assert abs(s - 1) < 0.001 + + +class MDP2(MDP): + """ + Inherits from MDP. Handles terminal states, and transitions to and from terminal states better. + """ + + def __init__(self, init, actlist, terminals, transitions, reward=None, gamma=0.9): + MDP.__init__(self, init, actlist, terminals, transitions, reward, gamma=gamma) + + def T(self, state, action): + """Return a list of (probability, next-state) pairs for taking the action in the + given state; a None action stays in place with probability 0.0.""" + if action is None: + return [(0.0, state)] + else: + return self.transitions[state][action] + + +class GridMDP(MDP): + """A two-dimensional grid MDP, as in [Figure 17.1]. All you have to do is + specify the grid as a list of lists of rewards; use None for an obstacle + (unreachable state). Also, you should specify the terminal states. + An action is an (x, y) unit vector; e.g. (1, 0) means move east.""" + + def __init__(self, grid, terminals, init=(0, 0), gamma=.9): + grid.reverse() # because we want row 0 on bottom, not on top + reward = {} + states = set() + self.rows = len(grid) + self.cols = len(grid[0]) + self.grid = grid + for x in range(self.cols): + for y in range(self.rows): + if grid[y][x]: + states.add((x, y)) + reward[(x, y)] = grid[y][x] + self.states = states + actlist = orientations + transitions = {} + for s in states: + transitions[s] = {} + for a in actlist: + transitions[s][a] = self.calculate_T(s, a) + MDP.__init__(self, init, actlist=actlist, + terminals=terminals, transitions=transitions, + reward=reward, states=states, gamma=gamma) + + def calculate_T(self, state, action): + """Return the (probability, next-state) outcomes for the intended action: 0.8 of + moving as intended and 0.1 each of veering to the right or left.""" + if action: + return [(0.8, self.go(state, action)), + (0.1, self.go(state, turn_right(action))), + (0.1, self.go(state, turn_left(action)))] + else: + return [(0.0, state)] + + def T(self, state, action): + """Return the list of (probability, next-state) pairs for the action in the given + state; a falsy action stays in place with probability 0.0.""" + return self.transitions[state][action] if action else [(0.0, state)] + + def go(self, state, direction): + """Return the state that results from going in this direction.""" + + state1 = vector_add(state, direction) + return state1 if state1 in self.states else state + + def to_grid(self, mapping): + """Convert a mapping from (x, y) to v into a [[..., v, ...]] grid.""" + + return list(reversed([[mapping.get((x, y), None) + for x in range(self.cols)] + for y in range(self.rows)])) + + def to_arrows(self, policy): + """Render a policy as a grid of arrow characters (``>``, ``^``, ``<``, ``v``, with + ``.`` for a None action) laid out to match the grid.""" + chars = {(1, 0): '>', (0, 1): '^', (-1, 0): '<', (0, -1): 'v', None: '.'} + return self.to_grid({s: chars[a] for (s, a) in policy.items()}) + + +def gen_grid(n_rows=3, n_cols=4, terminals=((3, 2), (3, 1)), main_reward=-0.04, + terminal_rewards=(1, -1), block_coords=((1, 1),)): + """Generate a grid (list of lists of rewards) of arbitrary size in the format + accepted by GridMDP, e.g. GridMDP(gen_grid(...), terminals=[...]). + n_rows, n_cols: grid dimensions. + terminals: (x, y) coordinates of the terminal cells. + main_reward: reward for every non-terminal, non-blocked cell. + terminal_rewards: reward for each cell in terminals (paired by position). + block_coords: (x, y) coordinates of obstacles (set to None / unreachable).""" + grid = [[main_reward] * n_cols for _ in range(n_rows)] + for (x, y), reward in zip(terminals, terminal_rewards): + grid[y][x] = reward + for x, y in block_coords: + grid[y][x] = None + grid.reverse() # row 0 at the bottom, matching GridMDP's convention + return grid + + +# ______________________________________________________________________________ + + +""" [Figure 17.1] +A 4x3 grid environment that presents the agent with a sequential decision problem. +""" + +sequential_decision_environment = GridMDP([[-0.04, -0.04, -0.04, +1], + [-0.04, None, -0.04, -1], + [-0.04, -0.04, -0.04, -0.04]], + terminals=[(3, 2), (3, 1)]) + + +# ______________________________________________________________________________ + + +def q_value(mdp, s, a, U): + """Return the Q-value of taking action ``a`` in state ``s`` given the utility estimates + ``U``, i.e. the expected sum of the immediate reward and the discounted utility of the + successor states. With no action it falls back to the reward of ``s``.""" + if not a: + return mdp.R(s) + res = 0 + for p, s_prime in mdp.T(s, a): + res += p * (mdp.R(s) + mdp.gamma * U[s_prime]) + return res + + +def value_iteration(mdp, epsilon=0.001): + """Solving an MDP by value iteration. [Figure 17.4]""" + + U1 = {s: 0 for s in mdp.states} + R, T, gamma = mdp.R, mdp.T, mdp.gamma + while True: + U = U1.copy() + delta = 0 + for s in mdp.states: + U1[s] = R(s) + gamma * max(sum(p * U[s1] for (p, s1) in T(s, a)) + for a in mdp.actions(s)) + delta = max(delta, abs(U1[s] - U[s])) + if delta <= epsilon * (1 - gamma) / gamma: + return U + + +def best_policy(mdp, U): + """Given an MDP and a utility function U, determine the best policy, + as a mapping from state to action. [Equation 17.4]""" + + pi = {} + for s in mdp.states: + pi[s] = max(mdp.actions(s), key=lambda a: expected_utility(a, s, U, mdp)) + return pi + + +def expected_utility(a, s, U, mdp): + """The expected utility of doing a in state s, according to the MDP and U.""" + + return sum(p * U[s1] for (p, s1) in mdp.T(s, a)) + + +# ______________________________________________________________________________ + + +def policy_iteration(mdp): + """Solve an MDP by policy iteration [Figure 17.7]""" + + U = {s: 0 for s in mdp.states} + pi = {s: random.choice(mdp.actions(s)) for s in mdp.states} + while True: + U = policy_evaluation(pi, U, mdp) + unchanged = True + for s in mdp.states: + a = max(mdp.actions(s), key=lambda a: expected_utility(a, s, U, mdp)) + if a != pi[s]: + pi[s] = a + unchanged = False + if unchanged: + return pi + + +def policy_evaluation(pi, U, mdp, k=20): + """Return an updated utility mapping U from each state in the MDP to its + utility, using an approximation (modified policy iteration).""" + + R, T, gamma = mdp.R, mdp.T, mdp.gamma + for i in range(k): + for s in mdp.states: + U[s] = R(s) + gamma * sum(p * U[s1] for (p, s1) in T(s, pi[s])) + return U + + +class POMDP(MDP): + r"""A Partially Observable Markov Decision Process, defined by + a transition model P(s'\|s,a), actions A(s), a reward function R(s), + and a sensor model P(e\|s). We also keep track of a gamma value, + for use by algorithms. The transition and the sensor models + are defined as matrices. We also keep track of the possible states + and actions for each state. [Page 659].""" + + def __init__(self, actions, transitions=None, evidences=None, rewards=None, states=None, gamma=0.95): + """Initialize variables of the pomdp""" + + if not (0 < gamma <= 1): + raise ValueError('A POMDP must have 0 < gamma <= 1') + + self.states = states + self.actions = actions + + # transition model cannot be undefined + self.t_prob = transitions or {} + if not self.t_prob: + print('Warning: Transition model is undefined') + + # sensor model cannot be undefined + self.e_prob = evidences or {} + if not self.e_prob: + print('Warning: Sensor model is undefined') + + self.gamma = gamma + self.rewards = rewards + + def remove_dominated_plans(self, input_values): + """ + Remove dominated plans. + This method finds all the lines contributing to the + upper surface and removes those which don't. + """ + + values = [val for action in input_values for val in input_values[action]] + values.sort(key=lambda x: x[0], reverse=True) + + best = [values[0]] + y1_max = max(val[1] for val in values) + tgt = values[0] + prev_b = 0 + prev_ix = 0 + while tgt[1] != y1_max: + min_b = 1 + min_ix = 0 + for i in range(prev_ix + 1, len(values)): + if values[i][0] - tgt[0] + tgt[1] - values[i][1] != 0: + trans_b = (values[i][0] - tgt[0]) / (values[i][0] - tgt[0] + tgt[1] - values[i][1]) + if 0 <= trans_b <= 1 and trans_b > prev_b and trans_b < min_b: + min_b = trans_b + min_ix = i + prev_b = min_b + prev_ix = min_ix + tgt = values[min_ix] + best.append(tgt) + + return self.generate_mapping(best, input_values) + + def remove_dominated_plans_fast(self, input_values): + """ + Remove dominated plans using approximations. + Resamples the upper boundary at intervals of 100 and + finds the maximum values at these points. + """ + + values = [val for action in input_values for val in input_values[action]] + values.sort(key=lambda x: x[0], reverse=True) + + best = [] + sr = 100 + for i in range(sr + 1): + x = i / float(sr) + maximum = (values[0][1] - values[0][0]) * x + values[0][0] + tgt = values[0] + for value in values: + val = (value[1] - value[0]) * x + value[0] + if val > maximum: + maximum = val + tgt = value + + if all(any(tgt != v) for v in best): + best.append(np.array(tgt)) + + return self.generate_mapping(best, input_values) + + def generate_mapping(self, best, input_values): + """Generate mappings after removing dominated plans""" + + mapping = defaultdict(list) + for value in best: + for action in input_values: + if any(all(value == v) for v in input_values[action]): + mapping[action].append(value) + + return mapping + + def max_difference(self, U1, U2): + """Find maximum difference between two utility mappings""" + + for k, v in U1.items(): + sum1 = 0 + for element in U1[k]: + sum1 += sum(element) + sum2 = 0 + for element in U2[k]: + sum2 += sum(element) + return abs(sum1 - sum2) + + +class Matrix: + """Matrix operations class""" + + @staticmethod + def add(A, B): + """Add two matrices A and B""" + + res = [] + for i in range(len(A)): + row = [] + for j in range(len(A[0])): + row.append(A[i][j] + B[i][j]) + res.append(row) + return res + + @staticmethod + def scalar_multiply(a, B): + """Multiply scalar a to matrix B""" + + for i in range(len(B)): + for j in range(len(B[0])): + B[i][j] = a * B[i][j] + return B + + @staticmethod + def multiply(A, B): + """Multiply two matrices A and B element-wise""" + + matrix = [] + for i in range(len(A)): + row = [] + for j in range(len(A[0])): + row.append(A[i][j] * B[i][j]) + matrix.append(row) + + return matrix + + @staticmethod + def matmul(A, B): + """Inner-product of two matrices""" + + return [[sum(ele_a * ele_b for ele_a, ele_b in zip(row_a, col_b)) for col_b in list(zip(*B))] for row_a in A] + + @staticmethod + def transpose(A): + """Transpose a matrix""" + + return [list(i) for i in zip(*A)] + + +def pomdp_value_iteration(pomdp, epsilon=0.1): + """Solving a POMDP by value iteration.""" + + U = {'': [[0] * len(pomdp.states)]} + count = 0 + while True: + count += 1 + prev_U = U + values = [val for action in U for val in U[action]] + value_matxs = [] + for i in values: + for j in values: + value_matxs.append([i, j]) + + U1 = defaultdict(list) + for action in pomdp.actions: + for u in value_matxs: + u1 = Matrix.matmul(Matrix.matmul(pomdp.t_prob[int(action)], + Matrix.multiply(pomdp.e_prob[int(action)], Matrix.transpose(u))), + [[1], [1]]) + u1 = Matrix.add(Matrix.scalar_multiply(pomdp.gamma, Matrix.transpose(u1)), [pomdp.rewards[int(action)]]) + U1[action].append(u1[0]) + + U = pomdp.remove_dominated_plans_fast(U1) + # replace with U = pomdp.remove_dominated_plans(U1) for accurate calculations + + if count > 10: + if pomdp.max_difference(U, prev_U) < epsilon * (1 - pomdp.gamma) / pomdp.gamma: + return U + + +def update_belief(pomdp, belief, action, observation): + """ + [Equation 17.17] + POMDP filtering: update the belief state (a probability distribution over the + states) after executing 'action' and perceiving 'observation'. The prediction + step propagates the belief through the transition model and the update step + weights it by the sensor model, then renormalizes:: + + b'(s') = alpha * P(o | s') * sum_s P(s' | s, a) b(s) + """ + transition, sensor = pomdp.t_prob[int(action)], pomdp.e_prob[int(action)] + n = len(belief) + # prediction: b_pred(s') = sum_s b(s) P(s' | s, a) + predicted = [sum(belief[s] * transition[s][sp] for s in range(n)) for sp in range(n)] + # update: weight each predicted state by the likelihood of the observation + updated = [sensor[sp][observation] * predicted[sp] for sp in range(n)] + total = sum(updated) + return [b / total for b in updated] if total else predicted + + +def pomdp_lookahead(pomdp, belief, depth): + """ + [Section 17.5 - Dynamic Decision Networks] + Online decision making for a POMDP modeled as a dynamic decision network + (DDN): the network is projected 'depth' steps into the future and solved by + belief-state expectimax search. Decision nodes range over the belief states + and chance nodes branch over the possible observations, with the belief + updated by POMDP filtering ([Equation 17.17]) along each branch. Returns the + action that maximizes the expected discounted utility from 'belief'. + """ + + def utility(belief, depth): + if depth == 0: + return 0 + return max(q_value(belief, action, depth) for action in pomdp.actions) + + def q_value(belief, action, depth): + # expected immediate reward of taking 'action' in the current belief + reward = sum(belief[s] * pomdp.rewards[int(action)][s] for s in range(len(belief))) + # expectation over the possible observations of the next belief's utility + sensor = pomdp.e_prob[int(action)] + predicted = [sum(belief[s] * pomdp.t_prob[int(action)][s][sp] for s in range(len(belief))) + for sp in range(len(belief))] + future = 0 + for observation in range(len(sensor[0])): + # P(observation | belief, action) + p_o = sum(predicted[sp] * sensor[sp][observation] for sp in range(len(belief))) + if p_o > 0: + future += p_o * utility(update_belief(pomdp, belief, action, observation), depth - 1) + return reward + pomdp.gamma * future + + return max(pomdp.actions, key=lambda action: q_value(belief, action, depth)) + + +__doc__ += """ +>>> pi = best_policy(sequential_decision_environment, value_iteration(sequential_decision_environment, .01)) + +>>> sequential_decision_environment.to_arrows(pi) +[['>', '>', '>', '.'], ['^', None, '^', '.'], ['^', '>', '^', '<']] + +>>> from utils import print_table + +>>> print_table(sequential_decision_environment.to_arrows(pi)) +> > > . +^ None ^ . +^ > ^ < + +>>> print_table(sequential_decision_environment.to_arrows(policy_iteration(sequential_decision_environment))) +> > > . +^ None ^ . +^ > ^ < +""" # noqa + +""" +s = { 'a' : { 'plan1' : [(0.2, 'a'), (0.3, 'b'), (0.3, 'c'), (0.2, 'd')], + 'plan2' : [(0.4, 'a'), (0.15, 'b'), (0.45, 'c')], + 'plan3' : [(0.2, 'a'), (0.5, 'b'), (0.3, 'c')], + }, + 'b' : { 'plan1' : [(0.2, 'a'), (0.6, 'b'), (0.2, 'c'), (0.1, 'd')], + 'plan2' : [(0.6, 'a'), (0.2, 'b'), (0.1, 'c'), (0.1, 'd')], + 'plan3' : [(0.3, 'a'), (0.3, 'b'), (0.4, 'c')], + }, + 'c' : { 'plan1' : [(0.3, 'a'), (0.5, 'b'), (0.1, 'c'), (0.1, 'd')], + 'plan2' : [(0.5, 'a'), (0.3, 'b'), (0.1, 'c'), (0.1, 'd')], + 'plan3' : [(0.1, 'a'), (0.3, 'b'), (0.1, 'c'), (0.5, 'd')], + }, + } +""" diff --git a/q-learning-9b/aima/nlp.py b/q-learning-9b/aima/nlp.py new file mode 100644 index 0000000..32243bb --- /dev/null +++ b/q-learning-9b/aima/nlp.py @@ -0,0 +1,721 @@ +"""Natural Language Processing; Chart Parsing and PageRanking (Chapter 22-23)""" + +from collections import defaultdict +from aima.utils import weighted_choice +from aima.search import Problem +import urllib.request +import copy +import heapq +import re + + +# ______________________________________________________________________________ +# Grammars and Lexicons + + +def Rules(**rules): + """Create a dictionary mapping symbols to alternative sequences. + >>> Rules(A = "B C | D E") + {'A': [['B', 'C'], ['D', 'E']]} + """ + for (lhs, rhs) in rules.items(): + rules[lhs] = [alt.strip().split() for alt in rhs.split('|')] + return rules + + +def Lexicon(**rules): + """Create a dictionary mapping symbols to alternative words. + >>> Lexicon(Article = "the | a | an") + {'Article': ['the', 'a', 'an']} + """ + for (lhs, rhs) in rules.items(): + rules[lhs] = [word.strip() for word in rhs.split('|')] + return rules + + +class Grammar: + """A context-free grammar defined by a set of rewrite rules and a lexicon. + + Rules map non-terminal categories to alternative right-hand sides, while the + lexicon maps categories to the words that belong to them.""" + + def __init__(self, name, rules, lexicon): + """A grammar has a set of rules and a lexicon.""" + self.name = name + self.rules = rules + self.lexicon = lexicon + self.categories = defaultdict(list) + for lhs in lexicon: + for word in lexicon[lhs]: + self.categories[word].append(lhs) + + def rewrites_for(self, cat): + """Return a sequence of possible rhs's that cat can be rewritten as.""" + return self.rules.get(cat, ()) + + def isa(self, word, cat): + """Return True iff word is of category cat""" + return cat in self.categories[word] + + def cnf_rules(self): + """Returns the tuple (X, Y, Z) for rules in the form: + X -> Y Z""" + cnf = [] + for X, rules in self.rules.items(): + for (Y, Z) in rules: + cnf.append((X, Y, Z)) + + return cnf + + def generate_random(self, S='S'): + """Replace each token in S by a random entry in grammar (recursively).""" + import random + + def rewrite(tokens, into): + for token in tokens: + if token in self.rules: + rewrite(random.choice(self.rules[token]), into) + elif token in self.lexicon: + into.append(random.choice(self.lexicon[token])) + else: + into.append(token) + return into + + return ' '.join(rewrite(S.split(), [])) + + def __repr__(self): + return ''.format(self.name) + + +def ProbRules(**rules): + """Create a dictionary mapping symbols to alternative sequences, + with probabilities. + >>> ProbRules(A = "B C [0.3] | D E [0.7]") + {'A': [(['B', 'C'], 0.3), (['D', 'E'], 0.7)]} + """ + for (lhs, rhs) in rules.items(): + rules[lhs] = [] + rhs_separate = [alt.strip().split() for alt in rhs.split('|')] + for r in rhs_separate: + prob = float(r[-1][1:-1]) # remove brackets, convert to float + rhs_rule = (r[:-1], prob) + rules[lhs].append(rhs_rule) + + return rules + + +def ProbLexicon(**rules): + """Create a dictionary mapping symbols to alternative words, + with probabilities. + >>> ProbLexicon(Article = "the [0.5] | a [0.25] | an [0.25]") + {'Article': [('the', 0.5), ('a', 0.25), ('an', 0.25)]} + """ + for (lhs, rhs) in rules.items(): + rules[lhs] = [] + rhs_separate = [word.strip().split() for word in rhs.split('|')] + for r in rhs_separate: + prob = float(r[-1][1:-1]) # remove brackets, convert to float + word = r[:-1][0] + rhs_rule = (word, prob) + rules[lhs].append(rhs_rule) + + return rules + + +class ProbGrammar: + """A probabilistic context-free grammar. + + Like :class:`Grammar`, but every rule and lexicon entry carries a + probability, so derivations can be sampled and scored by likelihood.""" + + def __init__(self, name, rules, lexicon): + """A grammar has a set of rules and a lexicon. + Each rule has a probability.""" + self.name = name + self.rules = rules + self.lexicon = lexicon + self.categories = defaultdict(list) + + for lhs in lexicon: + for word, prob in lexicon[lhs]: + self.categories[word].append((lhs, prob)) + + def rewrites_for(self, cat): + """Return a sequence of possible rhs's that cat can be rewritten as.""" + return self.rules.get(cat, ()) + + def isa(self, word, cat): + """Return True iff word is of category cat""" + return cat in [c for c, _ in self.categories[word]] + + def cnf_rules(self): + """Returns the tuple (X, Y, Z, p) for rules in the form: + X -> Y Z [p]""" + cnf = [] + for X, rules in self.rules.items(): + for (Y, Z), p in rules: + cnf.append((X, Y, Z, p)) + + return cnf + + def generate_random(self, S='S'): + """Replace each token in S by a random entry in grammar (recursively). + Returns a tuple of (sentence, probability).""" + import random + + def rewrite(tokens, into): + for token in tokens: + if token in self.rules: + non_terminal, prob = weighted_choice(self.rules[token]) + into[1] *= prob + rewrite(non_terminal, into) + elif token in self.lexicon: + terminal, prob = weighted_choice(self.lexicon[token]) + into[0].append(terminal) + into[1] *= prob + else: + into[0].append(token) + return into + + rewritten_as, prob = rewrite(S.split(), [[], 1]) + return (' '.join(rewritten_as), prob) + + def __repr__(self): + return ''.format(self.name) + + +E0 = Grammar('E0', + Rules( # Grammar for E_0 [Figure 22.4] + S='NP VP | S Conjunction S', + NP='Pronoun | Name | Noun | Article Noun | Digit Digit | NP PP | NP RelClause', + VP='Verb | VP NP | VP Adjective | VP PP | VP Adverb', + PP='Preposition NP', + RelClause='That VP'), + + Lexicon( # Lexicon for E_0 [Figure 22.3] + Noun="stench | breeze | glitter | nothing | wumpus | pit | pits | gold | east", + Verb="is | see | smell | shoot | fell | stinks | go | grab | carry | kill | turn | feel", # noqa + Adjective="right | left | east | south | back | smelly", + Adverb="here | there | nearby | ahead | right | left | east | south | back", + Pronoun="me | you | I | it", + Name="John | Mary | Boston | Aristotle", + Article="the | a | an", + Preposition="to | in | on | near", + Conjunction="and | or | but", + Digit="0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9", + That="that" + )) + +E_ = Grammar('E_', # Trivial Grammar and lexicon for testing + Rules( + S='NP VP', + NP='Art N | Pronoun', + VP='V NP'), + + Lexicon( + Art='the | a', + N='man | woman | table | shoelace | saw', + Pronoun='I | you | it', + V='saw | liked | feel' + )) + +E_NP_ = Grammar('E_NP_', # Another Trivial Grammar for testing + Rules(NP='Adj NP | N'), + Lexicon(Adj='happy | handsome | hairy', + N='man')) + +E_Prob = ProbGrammar('E_Prob', # The Probabilistic Grammar from the notebook + ProbRules( + S="NP VP [0.6] | S Conjunction S [0.4]", + NP="Pronoun [0.2] | Name [0.05] | Noun [0.2] | Article Noun [0.15] \ + | Article Adjs Noun [0.1] | Digit [0.05] | NP PP [0.15] | NP RelClause [0.1]", + VP="Verb [0.3] | VP NP [0.2] | VP Adjective [0.25] | VP PP [0.15] | VP Adverb [0.1]", + Adjs="Adjective [0.5] | Adjective Adjs [0.5]", + PP="Preposition NP [1]", + RelClause="RelPro VP [1]" + ), + ProbLexicon( + Verb="is [0.5] | say [0.3] | are [0.2]", + Noun="robot [0.4] | sheep [0.4] | fence [0.2]", + Adjective="good [0.5] | new [0.2] | sad [0.3]", + Adverb="here [0.6] | lightly [0.1] | now [0.3]", + Pronoun="me [0.3] | you [0.4] | he [0.3]", + RelPro="that [0.5] | who [0.3] | which [0.2]", + Name="john [0.4] | mary [0.4] | peter [0.2]", + Article="the [0.5] | a [0.25] | an [0.25]", + Preposition="to [0.4] | in [0.3] | at [0.3]", + Conjunction="and [0.5] | or [0.2] | but [0.3]", + Digit="0 [0.35] | 1 [0.35] | 2 [0.3]" + )) + +E_Chomsky = Grammar('E_Prob_Chomsky', # A Grammar in Chomsky Normal Form + Rules( + S='NP VP', + NP='Article Noun | Adjective Noun', + VP='Verb NP | Verb Adjective', + ), + Lexicon( + Article='the | a | an', + Noun='robot | sheep | fence', + Adjective='good | new | sad', + Verb='is | say | are' + )) + +E_Prob_Chomsky = ProbGrammar('E_Prob_Chomsky', # A Probabilistic Grammar in CNF + ProbRules( + S='NP VP [1]', + NP='Article Noun [0.6] | Adjective Noun [0.4]', + VP='Verb NP [0.5] | Verb Adjective [0.5]', + ), + ProbLexicon( + Article='the [0.5] | a [0.25] | an [0.25]', + Noun='robot [0.4] | sheep [0.4] | fence [0.2]', + Adjective='good [0.5] | new [0.2] | sad [0.3]', + Verb='is [0.5] | say [0.3] | are [0.2]' + )) +E_Prob_Chomsky_ = ProbGrammar('E_Prob_Chomsky_', + ProbRules( + S='NP VP [1]', + NP='NP PP [0.4] | Noun Verb [0.6]', + PP='Preposition NP [1]', + VP='Verb NP [0.7] | VP PP [0.3]', + ), + ProbLexicon( + Noun='astronomers [0.18] | eyes [0.32] | stars [0.32] | telescopes [0.18]', + Verb='saw [0.5] | \'\' [0.5]', + Preposition='with [1]' + )) + + +# ______________________________________________________________________________ +# Chart Parsing + + +class Chart: + """Class for parsing sentences using a chart data structure. + >>> chart = Chart(E0) + >>> len(chart.parses('the stench is in 2 2')) + 1 + """ + + def __init__(self, grammar, trace=False): + """A datastructure for parsing a string; and methods to do the parse. + self.chart[i] holds the edges that end just before the i'th word. + Edges are 5-element lists of [start, end, lhs, [found], [expects]].""" + self.grammar = grammar + self.trace = trace + + def parses(self, words, S='S'): + """Return a list of parses; words can be a list or string.""" + if isinstance(words, str): + words = words.split() + self.parse(words, S) + # Return all the parses that span the whole input + # 'span the whole input' => begin at 0, end at len(words) + return [[i, j, S, found, []] + for (i, j, lhs, found, expects) in self.chart[len(words)] + # assert j == len(words) + if i == 0 and lhs == S and expects == []] + + def parse(self, words, S='S'): + """Parse a list of words; according to the grammar. + Leave results in the chart.""" + self.chart = [[] for i in range(len(words) + 1)] + self.add_edge([0, 0, 'S_', [], [S]]) + for i in range(len(words)): + self.scanner(i, words[i]) + return self.chart + + def add_edge(self, edge): + """Add edge to chart, and see if it extends or predicts another edge.""" + start, end, lhs, found, expects = edge + if edge not in self.chart[end]: + self.chart[end].append(edge) + if self.trace: + print('Chart: added {}'.format(edge)) + if not expects: + self.extender(edge) + else: + self.predictor(edge) + + def scanner(self, j, word): + """For each edge expecting a word of this category here, extend the edge.""" + for (i, j, A, alpha, Bb) in self.chart[j]: + if Bb and self.grammar.isa(word, Bb[0]): + self.add_edge([i, j + 1, A, alpha + [(Bb[0], word)], Bb[1:]]) + + def predictor(self, edge): + """Add to chart any rules for B that could help extend this edge.""" + (i, j, A, alpha, Bb) = edge + B = Bb[0] + if B in self.grammar.rules: + for rhs in self.grammar.rewrites_for(B): + self.add_edge([j, j, B, [], rhs]) + + def extender(self, edge): + """See what edges can be extended by this edge.""" + (j, k, B, _, _) = edge + for (i, j, A, alpha, B1b) in self.chart[j]: + if B1b and B == B1b[0]: + self.add_edge([i, k, A, alpha + [edge], B1b[1:]]) + + +# ______________________________________________________________________________ +# CYK Parsing + +def CYK_parse(words, grammar): + """ [Figure 23.5] """ + # We use 0-based indexing instead of the book's 1-based. + N = len(words) + P = defaultdict(float) + + # Insert lexical rules for each word. + for (i, word) in enumerate(words): + for (X, p) in grammar.categories[word]: + P[X, i, 1] = p + + # Combine first and second parts of right-hand sides of rules, + # from short to long. + for length in range(2, N + 1): + for start in range(N - length + 1): + for len1 in range(1, length): # N.B. the book incorrectly has N instead of length + len2 = length - len1 + for (X, Y, Z, p) in grammar.cnf_rules(): + P[X, start, length] = max(P[X, start, length], + P[Y, start, len1] * P[Z, start + len1, len2] * p) + + return P + + +class Tree: + """A simple parse-tree node with a root label and a list of child leaves + (which may themselves be subtrees), as built by CYK parsing.""" + + def __init__(self, root, *args): + self.root = root + self.leaves = [leaf for leaf in args] + + +def subspan(N): + """returns all tuple(i, j, k) covering a span (i, k) with i <= j < k""" + for length in range(2, N + 1): + for i in range(1, N + 2 - length): + k = i + length - 1 + for j in range(i, k): + yield (i, j, k) + + +# ______________________________________________________________________________ +# Text Parsing using Search + + +class TextParsingProblem(Problem): + """A search problem that parses a list of words bottom-up into a goal symbol. + + States are partially-reduced word/category sequences; actions replace words + with their lexical categories and replace spans of categories by the rules + that produce them, so that a solution path corresponds to a valid parse.""" + + def __init__(self, initial, grammar, goal='S'): + """ + :param initial: the initial state of words in a list. + :param grammar: a grammar object + :param goal: the goal state, usually S + """ + super(TextParsingProblem, self).__init__(initial, goal) + self.grammar = grammar + self.combinations = defaultdict(list) # article combinations + # backward lookup of rules + for rule in grammar.rules: + for comb in grammar.rules[rule]: + self.combinations[' '.join(comb)].append(rule) + + def actions(self, state): + """Return the successor states reachable from ``state``: first replace each + word by each of its lexical categories, and once only categories remain + replace spans that match a grammar rule's right-hand side by that rule.""" + actions = [] + categories = self.grammar.categories + # first change each word to the article of its category + for i in range(len(state)): + word = state[i] + if word in categories: + for X in categories[word]: + state[i] = X + actions.append(copy.copy(state)) + state[i] = word + # if all words are replaced by articles, replace combinations of articles by inferring rules. + if not actions: + for start in range(len(state)): + for end in range(start, len(state) + 1): + # try combinations between (start, end) + articles = ' '.join(state[start:end]) + for c in self.combinations[articles]: + actions.append(state[:start] + [c] + state[end:]) + return actions + + def result(self, state, action): + """Return the state resulting from applying ``action`` (the action is itself + the already-computed successor state).""" + return action + + def h(self, state): + """Heuristic estimating remaining cost as the number of symbols still left + to reduce in ``state``.""" + # heuristic function + return len(state) + + +def astar_search_parsing(words, gramma): + """bottom-up parsing using A* search to find whether a list of words is a sentence""" + # init the problem + problem = TextParsingProblem(words, gramma, 'S') + state = problem.initial + # init the searching frontier + frontier = [(len(state) + problem.h(state), state)] + heapq.heapify(frontier) + + while frontier: + # search the frontier node with lowest cost first + cost, state = heapq.heappop(frontier) + actions = problem.actions(state) + for action in actions: + new_state = problem.result(state, action) + # update the new frontier node to the frontier + if new_state == [problem.goal]: + return problem.goal + if new_state != state: + heapq.heappush(frontier, (len(new_state) + problem.h(new_state), new_state)) + return False + + +def beam_search_parsing(words, gramma, b=3): + """bottom-up text parsing using beam search""" + # init problem + problem = TextParsingProblem(words, gramma, 'S') + # init frontier + frontier = [(len(problem.initial), problem.initial)] + heapq.heapify(frontier) + + # explore the current frontier and keep b new states with lowest cost + def explore(frontier): + new_frontier = [] + for cost, state in frontier: + # expand the possible children states of current state + if not problem.goal_test(' '.join(state)): + actions = problem.actions(state) + for action in actions: + new_state = problem.result(state, action) + if [len(new_state), new_state] not in new_frontier and new_state != state: + new_frontier.append([len(new_state), new_state]) + else: + return problem.goal + heapq.heapify(new_frontier) + # only keep b states + return heapq.nsmallest(b, new_frontier) + + while frontier: + frontier = explore(frontier) + if frontier == problem.goal: + return frontier + return False + + +# ______________________________________________________________________________ +# Page Ranking + +# First entry in list is the base URL, and then following are relative URL pages +example_pages_set = ["https://en.wikipedia.org/wiki/", "Aesthetics", "Analytic_philosophy", + "Ancient_Greek", "Aristotle", "Astrology", "Atheism", "Baruch_Spinoza", + "Belief", "Betrand Russell", "Confucius", "Consciousness", + "Continental Philosophy", "Dialectic", "Eastern_Philosophy", + "Epistemology", "Ethics", "Existentialism", "Friedrich_Nietzsche", + "Idealism", "Immanuel_Kant", "List_of_political_philosophers", "Logic", + "Metaphysics", "Philosophers", "Philosophy", "Philosophy_of_mind", "Physics", + "Plato", "Political_philosophy", "Pythagoras", "Rationalism", + "Social_philosophy", "Socrates", "Subjectivity", "Theology", + "Truth", "Western_philosophy"] + + +def load_page_html(address_list): + """Download HTML page content for every URL address passed as argument""" + content_dict = {} + for addr in address_list: + with urllib.request.urlopen(addr) as response: + raw_html = response.read().decode('utf-8') + # Strip raw html of unnecessary content. Basically everything that isn't link or text + html = strip_raw_html(raw_html) + content_dict[addr] = html + return content_dict + + +def init_pages(address_list): + """Create a dictionary of pages from a list of URL addresses""" + pages = {} + for addr in address_list: + pages[addr] = Page(addr) + return pages + + +def strip_raw_html(raw_html): + """Remove the section of the HTML which contains links to stylesheets etc., + and remove all other unnecessary HTML""" + # TODO: Strip more out of the raw html + return re.sub(".*?", "", raw_html, flags=re.DOTALL) # remove section + + +def determine_inlinks(page): + """Given a set of pages that have their outlinks determined, we can fill + out a page's inlinks by looking through all other page's outlinks""" + inlinks = [] + for addr, index_page in pages_index.items(): + if page.address == index_page.address: + continue + elif page.address in index_page.outlinks: + inlinks.append(addr) + return inlinks + + +def find_outlinks(page, handle_urls=None): + """Search a page's HTML content for URL links to other pages""" + urls = re.findall(r'href=[\'"]?([^\'" >]+)', pages_content[page.address]) + if handle_urls: + urls = handle_urls(urls) + return urls + + +def only_wikipedia_urls(urls): + """Some example HTML page data is from wikipedia. This function converts + relative wikipedia links to full wikipedia URLs""" + wiki_urls = [url for url in urls if url.startswith('/wiki/')] + return ["https://en.wikipedia.org" + url for url in wiki_urls] + + +# ______________________________________________________________________________ +# HITS Helper Functions + +def expand_pages(pages): + """Adds in every page that links to or is linked from one of + the relevant pages.""" + expanded = {} + for addr, page in pages.items(): + if addr not in expanded: + expanded[addr] = page + for inlink in page.inlinks: + if inlink not in expanded: + expanded[inlink] = pages_index[inlink] + for outlink in page.outlinks: + if outlink not in expanded: + expanded[outlink] = pages_index[outlink] + return expanded + + +def relevant_pages(query): + """Relevant pages are pages that contain all of the query words. They are obtained by + intersecting the hit lists of the query words.""" + hit_intersection = {addr for addr in pages_index} + query_words = query.split() + for query_word in query_words: + hit_list = set() + for addr in pages_index: + if query_word.lower() in pages_content[addr].lower(): + hit_list.add(addr) + hit_intersection = hit_intersection.intersection(hit_list) + return {addr: pages_index[addr] for addr in hit_intersection} + + +def normalize(pages): + """Normalize divides each page's score by the sum of the squares of all + pages' scores (separately for both the authority and hub scores). + """ + summed_hub = sum(page.hub ** 2 for _, page in pages.items()) + summed_auth = sum(page.authority ** 2 for _, page in pages.items()) + for _, page in pages.items(): + page.hub /= summed_hub ** 0.5 + page.authority /= summed_auth ** 0.5 + + +class ConvergenceDetector(object): + """If the hub and authority values of the pages are no longer changing, we have + reached a convergence and further iterations will have no effect. This detects convergence + so that we can stop the HITS algorithm as early as possible.""" + + def __init__(self): + self.hub_history = None + self.auth_history = None + + def __call__(self): + return self.detect() + + def detect(self): + """Return True once the average change in hub and authority values across all + indexed pages falls below the convergence threshold, otherwise False.""" + curr_hubs = [page.hub for addr, page in pages_index.items()] + curr_auths = [page.authority for addr, page in pages_index.items()] + if self.hub_history is None: + self.hub_history, self.auth_history = [], [] + else: + diffs_hub = [abs(x - y) for x, y in zip(curr_hubs, self.hub_history[-1])] + diffs_auth = [abs(x - y) for x, y in zip(curr_auths, self.auth_history[-1])] + ave_delta_hub = sum(diffs_hub) / float(len(pages_index)) + ave_delta_auth = sum(diffs_auth) / float(len(pages_index)) + if ave_delta_hub < 0.01 and ave_delta_auth < 0.01: # may need tweaking + return True + if len(self.hub_history) > 2: # prevent list from getting long + del self.hub_history[0] + del self.auth_history[0] + self.hub_history.append([x for x in curr_hubs]) + self.auth_history.append([x for x in curr_auths]) + return False + + +def get_in_links(page): + """Return the addresses of indexed pages that link into the given page.""" + if not page.inlinks: + page.inlinks = determine_inlinks(page) + return [addr for addr, p in pages_index.items() if addr in page.inlinks] + + +def get_out_links(page): + """Return the addresses of indexed pages that the given page links out to.""" + if not page.outlinks: + page.outlinks = find_outlinks(page) + return [addr for addr, p in pages_index.items() if addr in page.outlinks] + + +# ______________________________________________________________________________ +# HITS Algorithm + +class Page(object): + """A web page used by the HITS algorithm, holding its address, inbound and + outbound links, and its current hub and authority scores.""" + + def __init__(self, address, in_links=None, out_links=None, hub=0, authority=0): + self.address = address + self.hub = hub + self.authority = authority + self.inlinks = in_links + self.outlinks = out_links + + +pages_content = {} # maps Page relative or absolute URL/location to page's HTML content +pages_index = {} +convergence = ConvergenceDetector() # assign function to variable to mimic pseudocode's syntax + + +def HITS(query): + """The HITS algorithm for computing hubs and authorities with respect to a query.""" + pages = expand_pages(relevant_pages(query)) + for p in pages.values(): + p.authority = 1 + p.hub = 1 + while not convergence(): + authority = {p: pages[p].authority for p in pages} + hub = {p: pages[p].hub for p in pages} + for p in pages: + # p.authority ← ∑i Inlinki(p).Hub + pages[p].authority = sum(hub[x] for x in get_in_links(pages[p])) + # p.hub ← ∑i Outlinki(p).Authority + pages[p].hub = sum(authority[x] for x in get_out_links(pages[p])) + normalize(pages) + return pages diff --git a/q-learning-9b/aima/notebook_utils.py b/q-learning-9b/aima/notebook_utils.py new file mode 100644 index 0000000..7b881d2 --- /dev/null +++ b/q-learning-9b/aima/notebook_utils.py @@ -0,0 +1,1311 @@ +import heapq +import os +import time +from collections import defaultdict +from inspect import getsource + +import ipywidgets as widgets +import matplotlib.pyplot as plt +import networkx as nx +import numpy as np +from IPython.display import HTML +from IPython.display import display +from PIL import Image +from matplotlib import lines +from matplotlib.colors import ListedColormap + +from aima.games import TicTacToe, alpha_beta_player, random_player, Fig52Extended +from aima.learning import DataSet +from aima.logic import parse_definite_clause, standardize_variables, unify_mm, subst +from aima.search import GraphProblem, romania_map, Node + +# repo root (the directory containing the `aima` package), so cwd-relative +# data/image paths resolve regardless of where a notebook is launched from +_AIMA_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# ______________________________________________________________________________ +# Magic Words + + +def pseudocode(algorithm): + """Print the pseudocode for the given algorithm.""" + from urllib.request import urlopen + from IPython.display import Markdown + + algorithm = algorithm.replace(' ', '-') + url = "https://raw.githubusercontent.com/aimacode/aima-pseudocode/master/md/{}.md".format(algorithm) + f = urlopen(url) + md = f.read().decode('utf-8') + md = md.split('\n', 1)[-1].strip() + md = '#' + md + return Markdown(md) + + +def psource(*functions): + """Print the source code for the given function(s).""" + source_code = '\n\n'.join(getsource(fn) for fn in functions) + try: + from pygments.formatters import HtmlFormatter + from pygments.lexers import PythonLexer + from pygments import highlight + + display(HTML(highlight(source_code, PythonLexer(), HtmlFormatter(full=True)))) + + except ImportError: + print(source_code) + + +# ______________________________________________________________________________ +# Iris Visualization + + +def show_iris(i=0, j=1, k=2): + """Plots the iris dataset in a 3D plot. + The three axes are given by i, j and k, + which correspond to three of the four iris features.""" + + plt.rcParams.update(plt.rcParamsDefault) + + fig = plt.figure() + ax = fig.add_subplot(111, projection='3d') + + iris = DataSet(name="iris") + buckets = iris.split_values_by_classes() + + features = ["Sepal Length", "Sepal Width", "Petal Length", "Petal Width"] + f1, f2, f3 = features[i], features[j], features[k] + + a_setosa = [v[i] for v in buckets["setosa"]] + b_setosa = [v[j] for v in buckets["setosa"]] + c_setosa = [v[k] for v in buckets["setosa"]] + + a_virginica = [v[i] for v in buckets["virginica"]] + b_virginica = [v[j] for v in buckets["virginica"]] + c_virginica = [v[k] for v in buckets["virginica"]] + + a_versicolor = [v[i] for v in buckets["versicolor"]] + b_versicolor = [v[j] for v in buckets["versicolor"]] + c_versicolor = [v[k] for v in buckets["versicolor"]] + + for c, m, sl, sw, pl in [('b', 's', a_setosa, b_setosa, c_setosa), + ('g', '^', a_virginica, b_virginica, c_virginica), + ('r', 'o', a_versicolor, b_versicolor, c_versicolor)]: + ax.scatter(sl, sw, pl, c=c, marker=m) + + ax.set_xlabel(f1) + ax.set_ylabel(f2) + ax.set_zlabel(f3) + + plt.show() + + +# ______________________________________________________________________________ +# MNIST + + +def load_MNIST(path="aima-data/MNIST/Digits", fashion=False): + """Load the MNIST (or Fashion-MNIST when ``fashion`` is True) dataset from its + IDX files under ``path``. Returns ``(train_img, train_lbl, test_img, test_lbl)`` + as numpy arrays, with images flattened to one row per sample. + """ + import os, struct + import array + import numpy as np + + if fashion: + path = "aima-data/MNIST/Fashion" + + if not os.path.isabs(path): + path = os.path.join(_AIMA_ROOT, path) + + plt.rcParams.update(plt.rcParamsDefault) + plt.rcParams['figure.figsize'] = (10.0, 8.0) + plt.rcParams['image.interpolation'] = 'nearest' + plt.rcParams['image.cmap'] = 'gray' + + train_img_file = open(os.path.join(path, "train-images-idx3-ubyte"), "rb") + train_lbl_file = open(os.path.join(path, "train-labels-idx1-ubyte"), "rb") + test_img_file = open(os.path.join(path, "t10k-images-idx3-ubyte"), "rb") + test_lbl_file = open(os.path.join(path, 't10k-labels-idx1-ubyte'), "rb") + + magic_nr, tr_size, tr_rows, tr_cols = struct.unpack(">IIII", train_img_file.read(16)) + tr_img = array.array("B", train_img_file.read()) + train_img_file.close() + magic_nr, tr_size = struct.unpack(">II", train_lbl_file.read(8)) + tr_lbl = array.array("b", train_lbl_file.read()) + train_lbl_file.close() + + magic_nr, te_size, te_rows, te_cols = struct.unpack(">IIII", test_img_file.read(16)) + te_img = array.array("B", test_img_file.read()) + test_img_file.close() + magic_nr, te_size = struct.unpack(">II", test_lbl_file.read(8)) + te_lbl = array.array("b", test_lbl_file.read()) + test_lbl_file.close() + + # print(len(tr_img), len(tr_lbl), tr_size) + # print(len(te_img), len(te_lbl), te_size) + + train_img = np.zeros((tr_size, tr_rows * tr_cols), dtype=np.int16) + train_lbl = np.zeros((tr_size,), dtype=np.int8) + for i in range(tr_size): + train_img[i] = np.array(tr_img[i * tr_rows * tr_cols: (i + 1) * tr_rows * tr_cols]).reshape((tr_rows * te_cols)) + train_lbl[i] = tr_lbl[i] + + test_img = np.zeros((te_size, te_rows * te_cols), dtype=np.int16) + test_lbl = np.zeros((te_size,), dtype=np.int8) + for i in range(te_size): + test_img[i] = np.array(te_img[i * te_rows * te_cols: (i + 1) * te_rows * te_cols]).reshape((te_rows * te_cols)) + test_lbl[i] = te_lbl[i] + + return (train_img, train_lbl, test_img, test_lbl) + + +digit_classes = [str(i) for i in range(10)] +fashion_classes = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", + "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"] + + +def show_MNIST(labels, images, samples=8, fashion=False): + """Display a grid of ``samples`` random example images for each class of the + (Fashion-)MNIST dataset. + """ + if not fashion: + classes = digit_classes + else: + classes = fashion_classes + + num_classes = len(classes) + + for y, cls in enumerate(classes): + idxs = np.nonzero([i == y for i in labels]) + idxs = np.random.choice(idxs[0], samples, replace=False) + for i, idx in enumerate(idxs): + plt_idx = i * num_classes + y + 1 + plt.subplot(samples, num_classes, plt_idx) + plt.imshow(images[idx].reshape((28, 28))) + plt.axis("off") + if i == 0: + plt.title(cls) + + plt.show() + + +def show_ave_MNIST(labels, images, fashion=False): + """Display the average image of every class in the (Fashion-)MNIST dataset and + print how many images each class contains. + """ + if not fashion: + item_type = "Digit" + classes = digit_classes + else: + item_type = "Apparel" + classes = fashion_classes + + num_classes = len(classes) + + for y, cls in enumerate(classes): + idxs = np.nonzero([i == y for i in labels]) + print(item_type, y, ":", len(idxs[0]), "images.") + + ave_img = np.mean(np.vstack([images[i] for i in idxs[0]]), axis=0) + # print(ave_img.shape) + + plt.subplot(1, num_classes, y + 1) + plt.imshow(ave_img.reshape((28, 28))) + plt.axis("off") + plt.title(cls) + + plt.show() + + +# ______________________________________________________________________________ +# MDP + + +def make_plot_grid_step_function(columns, rows, U_over_time): + """ipywidgets interactive function supports single parameter as input. + This function creates and return such a function by taking as input + other parameters.""" + + def plot_grid_step(iteration): + data = U_over_time[iteration] + data = defaultdict(lambda: 0, data) + grid = [] + for row in range(rows): + current_row = [] + for column in range(columns): + current_row.append(data[(column, row)]) + grid.append(current_row) + grid.reverse() # output like book + fig = plt.imshow(grid, cmap=plt.cm.bwr, interpolation='nearest') + + plt.axis('off') + fig.axes.get_xaxis().set_visible(False) + fig.axes.get_yaxis().set_visible(False) + + for col in range(len(grid)): + for row in range(len(grid[0])): + magic = grid[col][row] + fig.axes.text(row, col, "{0:.2f}".format(magic), va='center', ha='center') + + plt.show() + + return plot_grid_step + + +def make_visualize(slider): + """Takes an input a sliderand returns callback function + for timer and animation.""" + + def visualize_callback(visualize, time_step): + if visualize is True: + for i in range(slider.min, slider.max + 1): + slider.value = i + time.sleep(float(time_step)) + + return visualize_callback + + +# ______________________________________________________________________________ + + +_canvas = """ + +
+ +
+ + +""" # noqa + + +class Canvas: + """Inherit from this class to manage the HTML canvas element in jupyter notebooks. + To create an object of this class any_name_xyz = Canvas("any_name_xyz") + The first argument given must be the name of the object being created. + IPython must be able to reference the variable name that is being passed.""" + + def __init__(self, varname, width=800, height=600, cid=None): + self.name = varname + self.cid = cid or varname + self.width = width + self.height = height + self.html = _canvas.format(self.cid, self.width, self.height, self.name) + self.exec_list = [] + display_html(self.html) + + def mouse_click(self, x, y): + """Override this method to handle mouse click at position (x, y)""" + raise NotImplementedError + + def mouse_move(self, x, y): + """Override this method to handle mouse move at position (x, y)""" + raise NotImplementedError + + def execute(self, exec_str): + """Stores the command to be executed to a list which is used later during update()""" + if not isinstance(exec_str, str): + print("Invalid execution argument:", exec_str) + self.alert("Received invalid execution command format") + prefix = "{0}_canvas_object.".format(self.cid) + self.exec_list.append(prefix + exec_str + ';') + + def fill(self, r, g, b): + """Changes the fill color to a color in rgb format""" + self.execute("fill({0}, {1}, {2})".format(r, g, b)) + + def stroke(self, r, g, b): + """Changes the colors of line/strokes to rgb""" + self.execute("stroke({0}, {1}, {2})".format(r, g, b)) + + def strokeWidth(self, w): + """Changes the width of lines/strokes to 'w' pixels""" + self.execute("strokeWidth({0})".format(w)) + + def rect(self, x, y, w, h): + """Draw a rectangle with 'w' width, 'h' height and (x, y) as the top-left corner""" + self.execute("rect({0}, {1}, {2}, {3})".format(x, y, w, h)) + + def rect_n(self, xn, yn, wn, hn): + """Similar to rect(), but the dimensions are normalized to fall between 0 and 1""" + x = round(xn * self.width) + y = round(yn * self.height) + w = round(wn * self.width) + h = round(hn * self.height) + self.rect(x, y, w, h) + + def line(self, x1, y1, x2, y2): + """Draw a line from (x1, y1) to (x2, y2)""" + self.execute("line({0}, {1}, {2}, {3})".format(x1, y1, x2, y2)) + + def line_n(self, x1n, y1n, x2n, y2n): + """Similar to line(), but the dimensions are normalized to fall between 0 and 1""" + x1 = round(x1n * self.width) + y1 = round(y1n * self.height) + x2 = round(x2n * self.width) + y2 = round(y2n * self.height) + self.line(x1, y1, x2, y2) + + def arc(self, x, y, r, start, stop): + """Draw an arc with (x, y) as centre, 'r' as radius from angles 'start' to 'stop'""" + self.execute("arc({0}, {1}, {2}, {3}, {4})".format(x, y, r, start, stop)) + + def arc_n(self, xn, yn, rn, start, stop): + """Similar to arc(), but the dimensions are normalized to fall between 0 and 1 + The normalizing factor for radius is selected between width and height by + seeing which is smaller.""" + x = round(xn * self.width) + y = round(yn * self.height) + r = round(rn * min(self.width, self.height)) + self.arc(x, y, r, start, stop) + + def clear(self): + """Clear the HTML canvas""" + self.execute("clear()") + + def font(self, font): + """Changes the font of text""" + self.execute('font("{0}")'.format(font)) + + def text(self, txt, x, y, fill=True): + """Display a text at (x, y)""" + if fill: + self.execute('fill_text("{0}", {1}, {2})'.format(txt, x, y)) + else: + self.execute('stroke_text("{0}", {1}, {2})'.format(txt, x, y)) + + def text_n(self, txt, xn, yn, fill=True): + """Similar to text(), but with normalized coordinates""" + x = round(xn * self.width) + y = round(yn * self.height) + self.text(txt, x, y, fill) + + def alert(self, message): + """Immediately display an alert""" + display_html(''.format(message)) + + def update(self): + """Execute the JS code to execute the commands queued by execute()""" + exec_code = "" + self.exec_list = [] + display_html(exec_code) + + +def display_html(html_string): + """Render the given HTML string in the Jupyter notebook output.""" + display(HTML(html_string)) + + +################################################################################ + + +class Canvas_TicTacToe(Canvas): + """Play a 3x3 TicTacToe game on HTML canvas""" + + def __init__(self, varname, player_1='human', player_2='random', + width=300, height=350, cid=None): + valid_players = ('human', 'random', 'alpha_beta') + if player_1 not in valid_players or player_2 not in valid_players: + raise TypeError("Players must be one of {}".format(valid_players)) + super().__init__(varname, width, height, cid) + self.ttt = TicTacToe() + self.state = self.ttt.initial + self.turn = 0 + self.strokeWidth(5) + self.players = (player_1, player_2) + self.font("20px Arial") + self.draw_board() + + def mouse_click(self, x, y): + """Handle a click at pixel (x, y): make the move for the current human or AI + player, or restart the game when it is over, then redraw the board. + """ + player = self.players[self.turn] + if self.ttt.terminal_test(self.state): + if 0.55 <= x / self.width <= 0.95 and 6 / 7 <= y / self.height <= 6 / 7 + 1 / 8: + self.state = self.ttt.initial + self.turn = 0 + self.draw_board() + return + + if player == 'human': + x, y = int(3 * x / self.width) + 1, int(3 * y / (self.height * 6 / 7)) + 1 + if (x, y) not in self.ttt.actions(self.state): + # Invalid move + return + move = (x, y) + elif player == 'alpha_beta': + move = alpha_beta_player(self.ttt, self.state) + else: + move = random_player(self.ttt, self.state) + self.state = self.ttt.result(self.state, move) + self.turn ^= 1 + self.draw_board() + + def draw_board(self): + """Redraw the board: grid lines, the current X/O marks, and the game status + (whose turn it is, the winning line, or the restart button). + """ + self.clear() + self.stroke(0, 0, 0) + offset = 1 / 20 + self.line_n(0 + offset, (1 / 3) * 6 / 7, 1 - offset, (1 / 3) * 6 / 7) + self.line_n(0 + offset, (2 / 3) * 6 / 7, 1 - offset, (2 / 3) * 6 / 7) + self.line_n(1 / 3, (0 + offset) * 6 / 7, 1 / 3, (1 - offset) * 6 / 7) + self.line_n(2 / 3, (0 + offset) * 6 / 7, 2 / 3, (1 - offset) * 6 / 7) + + board = self.state.board + for mark in board: + if board[mark] == 'X': + self.draw_x(mark) + elif board[mark] == 'O': + self.draw_o(mark) + if self.ttt.terminal_test(self.state): + # End game message + utility = self.ttt.utility(self.state, self.ttt.to_move(self.ttt.initial)) + if utility == 0: + self.text_n('Game Draw!', offset, 6 / 7 + offset) + else: + self.text_n('Player {} wins!'.format("XO"[utility < 0]), offset, 6 / 7 + offset) + # Find the 3 and draw a line + self.stroke([255, 0][self.turn], [0, 255][self.turn], 0) + for i in range(3): + if all([(i + 1, j + 1) in self.state.board for j in range(3)]) and \ + len({self.state.board[(i + 1, j + 1)] for j in range(3)}) == 1: + self.line_n(i / 3 + 1 / 6, offset * 6 / 7, i / 3 + 1 / 6, (1 - offset) * 6 / 7) + if all([(j + 1, i + 1) in self.state.board for j in range(3)]) and \ + len({self.state.board[(j + 1, i + 1)] for j in range(3)}) == 1: + self.line_n(offset, (i / 3 + 1 / 6) * 6 / 7, 1 - offset, (i / 3 + 1 / 6) * 6 / 7) + if all([(i + 1, i + 1) in self.state.board for i in range(3)]) and \ + len({self.state.board[(i + 1, i + 1)] for i in range(3)}) == 1: + self.line_n(offset, offset * 6 / 7, 1 - offset, (1 - offset) * 6 / 7) + if all([(i + 1, 3 - i) in self.state.board for i in range(3)]) and \ + len({self.state.board[(i + 1, 3 - i)] for i in range(3)}) == 1: + self.line_n(offset, (1 - offset) * 6 / 7, 1 - offset, offset * 6 / 7) + # restart button + self.fill(0, 0, 255) + self.rect_n(0.5 + offset, 6 / 7, 0.4, 1 / 8) + self.fill(0, 0, 0) + self.text_n('Restart', 0.5 + 2 * offset, 13 / 14) + else: # Print which player's turn it is + self.text_n("Player {}'s move({})".format("XO"[self.turn], self.players[self.turn]), + offset, 6 / 7 + offset) + + self.update() + + def draw_x(self, position): + """Draw an 'X' mark in the cell at the given (column, row) board position.""" + self.stroke(0, 255, 0) + x, y = [i - 1 for i in position] + offset = 1 / 15 + self.line_n(x / 3 + offset, (y / 3 + offset) * 6 / 7, x / 3 + 1 / 3 - offset, (y / 3 + 1 / 3 - offset) * 6 / 7) + self.line_n(x / 3 + 1 / 3 - offset, (y / 3 + offset) * 6 / 7, x / 3 + offset, (y / 3 + 1 / 3 - offset) * 6 / 7) + + def draw_o(self, position): + """Draw an 'O' mark in the cell at the given (column, row) board position.""" + self.stroke(255, 0, 0) + x, y = [i - 1 for i in position] + self.arc_n(x / 3 + 1 / 6, (y / 3 + 1 / 6) * 6 / 7, 1 / 9, 0, 360) + + +class Canvas_min_max(Canvas): + """MinMax for Fig52Extended on HTML canvas""" + + def __init__(self, varname, util_list, width=800, height=600, cid=None): + super().__init__(varname, width, height, cid) + self.utils = {node: util for node, util in zip(range(13, 40), util_list)} + self.game = Fig52Extended() + self.game.utils = self.utils + self.nodes = list(range(40)) + self.l = 1 / 40 + self.node_pos = {} + for i in range(4): + base = len(self.node_pos) + row_size = 3 ** i + for node in [base + j for j in range(row_size)]: + self.node_pos[node] = ((node - base) / row_size + 1 / (2 * row_size) - self.l / 2, + self.l / 2 + (self.l + (1 - 5 * self.l) / 3) * i) + self.font("12px Arial") + self.node_stack = [] + self.explored = {node for node in self.utils} + self.thick_lines = set() + self.change_list = [] + self.draw_graph() + self.stack_manager = self.stack_manager_gen() + + def min_max(self, node): + """Run minimax from the given node, recording the sequence of canvas changes + (visited nodes, explored utilities, thick edges) for step-by-step animation. + Returns the node's minimax value. + """ + game = self.game + player = game.to_move(node) + + def max_value(node): + if game.terminal_test(node): + return game.utility(node, player) + self.change_list.append(('a', node)) + self.change_list.append(('h',)) + max_a = max(game.actions(node), key=lambda x: min_value(game.result(node, x))) + max_node = game.result(node, max_a) + self.utils[node] = self.utils[max_node] + x1, y1 = self.node_pos[node] + x2, y2 = self.node_pos[max_node] + self.change_list.append(('l', (node, max_node - 3 * node - 1))) + self.change_list.append(('e', node)) + self.change_list.append(('p',)) + self.change_list.append(('h',)) + return self.utils[node] + + def min_value(node): + if game.terminal_test(node): + return game.utility(node, player) + self.change_list.append(('a', node)) + self.change_list.append(('h',)) + min_a = min(game.actions(node), key=lambda x: max_value(game.result(node, x))) + min_node = game.result(node, min_a) + self.utils[node] = self.utils[min_node] + x1, y1 = self.node_pos[node] + x2, y2 = self.node_pos[min_node] + self.change_list.append(('l', (node, min_node - 3 * node - 1))) + self.change_list.append(('e', node)) + self.change_list.append(('p',)) + self.change_list.append(('h',)) + return self.utils[node] + + return max_value(node) + + def stack_manager_gen(self): + """Generator that replays the recorded change list, updating the node stack, + explored set and thick lines, and yielding once per animation step. + """ + self.min_max(0) + for change in self.change_list: + if change[0] == 'a': + self.node_stack.append(change[1]) + elif change[0] == 'e': + self.explored.add(change[1]) + elif change[0] == 'h': + yield + elif change[0] == 'l': + self.thick_lines.add(change[1]) + elif change[0] == 'p': + self.node_stack.pop() + + def mouse_click(self, x, y): + """Advance the minimax animation by one step on each click, then redraw the graph.""" + try: + self.stack_manager.send(None) + except StopIteration: + pass + self.draw_graph() + + def draw_graph(self): + """Draw the game tree: the nodes (highlighting those on the stack and showing the + utilities of explored nodes) together with the edges connecting them. + """ + self.clear() + # draw nodes + self.stroke(0, 0, 0) + self.strokeWidth(1) + # highlight for nodes in stack + for node in self.node_stack: + x, y = self.node_pos[node] + self.fill(200, 200, 0) + self.rect_n(x - self.l / 5, y - self.l / 5, self.l * 7 / 5, self.l * 7 / 5) + for node in self.nodes: + x, y = self.node_pos[node] + if node in self.explored: + self.fill(255, 255, 255) + else: + self.fill(200, 200, 200) + self.rect_n(x, y, self.l, self.l) + self.line_n(x, y, x + self.l, y) + self.line_n(x, y, x, y + self.l) + self.line_n(x + self.l, y + self.l, x + self.l, y) + self.line_n(x + self.l, y + self.l, x, y + self.l) + self.fill(0, 0, 0) + if node in self.explored: + self.text_n(self.utils[node], x + self.l / 10, y + self.l * 9 / 10) + # draw edges + for i in range(13): + x1, y1 = self.node_pos[i][0] + self.l / 2, self.node_pos[i][1] + self.l + for j in range(3): + x2, y2 = self.node_pos[i * 3 + j + 1][0] + self.l / 2, self.node_pos[i * 3 + j + 1][1] + if i in [1, 2, 3]: + self.stroke(200, 0, 0) + else: + self.stroke(0, 200, 0) + if (i, j) in self.thick_lines: + self.strokeWidth(3) + else: + self.strokeWidth(1) + self.line_n(x1, y1, x2, y2) + self.update() + + +class Canvas_alpha_beta(Canvas): + """Alpha-beta pruning for Fig52Extended on HTML canvas""" + + def __init__(self, varname, util_list, width=800, height=600, cid=None): + super().__init__(varname, width, height, cid) + self.utils = {node: util for node, util in zip(range(13, 40), util_list)} + self.game = Fig52Extended() + self.game.utils = self.utils + self.nodes = list(range(40)) + self.l = 1 / 40 + self.node_pos = {} + for i in range(4): + base = len(self.node_pos) + row_size = 3 ** i + for node in [base + j for j in range(row_size)]: + self.node_pos[node] = ((node - base) / row_size + 1 / (2 * row_size) - self.l / 2, + 3 * self.l / 2 + (self.l + (1 - 6 * self.l) / 3) * i) + self.font("12px Arial") + self.node_stack = [] + self.explored = {node for node in self.utils} + self.pruned = set() + self.ab = {} + self.thick_lines = set() + self.change_list = [] + self.draw_graph() + self.stack_manager = self.stack_manager_gen() + + def alpha_beta_search(self, node): + """Run alpha-beta pruning from the given node, recording the sequence of canvas + changes (visited nodes, alpha/beta bounds, pruned nodes, thick edges) for + step-by-step animation. Returns the node's value. + """ + game = self.game + player = game.to_move(node) + + # Functions used by alpha_beta + def max_value(node, alpha, beta): + if game.terminal_test(node): + self.change_list.append(('a', node)) + self.change_list.append(('h',)) + self.change_list.append(('p',)) + return game.utility(node, player) + v = -np.inf + self.change_list.append(('a', node)) + self.change_list.append(('ab', node, v, beta)) + self.change_list.append(('h',)) + for a in game.actions(node): + min_val = min_value(game.result(node, a), alpha, beta) + if v < min_val: + v = min_val + max_node = game.result(node, a) + self.change_list.append(('ab', node, v, beta)) + if v >= beta: + self.change_list.append(('h',)) + self.pruned.add(node) + break + alpha = max(alpha, v) + self.utils[node] = v + if node not in self.pruned: + self.change_list.append(('l', (node, max_node - 3 * node - 1))) + self.change_list.append(('e', node)) + self.change_list.append(('p',)) + self.change_list.append(('h',)) + return v + + def min_value(node, alpha, beta): + if game.terminal_test(node): + self.change_list.append(('a', node)) + self.change_list.append(('h',)) + self.change_list.append(('p',)) + return game.utility(node, player) + v = np.inf + self.change_list.append(('a', node)) + self.change_list.append(('ab', node, alpha, v)) + self.change_list.append(('h',)) + for a in game.actions(node): + max_val = max_value(game.result(node, a), alpha, beta) + if v > max_val: + v = max_val + min_node = game.result(node, a) + self.change_list.append(('ab', node, alpha, v)) + if v <= alpha: + self.change_list.append(('h',)) + self.pruned.add(node) + break + beta = min(beta, v) + self.utils[node] = v + if node not in self.pruned: + self.change_list.append(('l', (node, min_node - 3 * node - 1))) + self.change_list.append(('e', node)) + self.change_list.append(('p',)) + self.change_list.append(('h',)) + return v + + return max_value(node, -np.inf, np.inf) + + def stack_manager_gen(self): + """Generator that replays the recorded change list, updating the node stack, + alpha/beta bounds, explored and pruned sets and thick lines, yielding once + per animation step. + """ + self.alpha_beta_search(0) + for change in self.change_list: + if change[0] == 'a': + self.node_stack.append(change[1]) + elif change[0] == 'ab': + self.ab[change[1]] = change[2:] + elif change[0] == 'e': + self.explored.add(change[1]) + elif change[0] == 'h': + yield + elif change[0] == 'l': + self.thick_lines.add(change[1]) + elif change[0] == 'p': + self.node_stack.pop() + + def mouse_click(self, x, y): + """Advance the alpha-beta animation by one step on each click, then redraw the graph.""" + try: + self.stack_manager.send(None) + except StopIteration: + pass + self.draw_graph() + + def draw_graph(self): + """Draw the game tree with nodes (highlighting those on the stack, explored and + pruned) and edges, and display the alpha/beta bounds of nodes on the stack. + """ + self.clear() + # draw nodes + self.stroke(0, 0, 0) + self.strokeWidth(1) + # highlight for nodes in stack + for node in self.node_stack: + x, y = self.node_pos[node] + # alpha > beta + if node not in self.explored and self.ab[node][0] > self.ab[node][1]: + self.fill(200, 100, 100) + else: + self.fill(200, 200, 0) + self.rect_n(x - self.l / 5, y - self.l / 5, self.l * 7 / 5, self.l * 7 / 5) + for node in self.nodes: + x, y = self.node_pos[node] + if node in self.explored: + if node in self.pruned: + self.fill(50, 50, 50) + else: + self.fill(255, 255, 255) + else: + self.fill(200, 200, 200) + self.rect_n(x, y, self.l, self.l) + self.line_n(x, y, x + self.l, y) + self.line_n(x, y, x, y + self.l) + self.line_n(x + self.l, y + self.l, x + self.l, y) + self.line_n(x + self.l, y + self.l, x, y + self.l) + self.fill(0, 0, 0) + if node in self.explored and node not in self.pruned: + self.text_n(self.utils[node], x + self.l / 10, y + self.l * 9 / 10) + # draw edges + for i in range(13): + x1, y1 = self.node_pos[i][0] + self.l / 2, self.node_pos[i][1] + self.l + for j in range(3): + x2, y2 = self.node_pos[i * 3 + j + 1][0] + self.l / 2, self.node_pos[i * 3 + j + 1][1] + if i in [1, 2, 3]: + self.stroke(200, 0, 0) + else: + self.stroke(0, 200, 0) + if (i, j) in self.thick_lines: + self.strokeWidth(3) + else: + self.strokeWidth(1) + self.line_n(x1, y1, x2, y2) + # display alpha and beta + for node in self.node_stack: + if node not in self.explored: + x, y = self.node_pos[node] + alpha, beta = self.ab[node] + self.text_n(alpha, x - self.l / 2, y - self.l / 10) + self.text_n(beta, x + self.l, y - self.l / 10) + self.update() + + +class Canvas_fol_bc_ask(Canvas): + """fol_bc_ask() on HTML canvas""" + + def __init__(self, varname, kb, query, width=800, height=600, cid=None): + super().__init__(varname, width, height, cid) + self.kb = kb + self.query = query + self.l = 1 / 20 + self.b = 3 * self.l + bc_out = list(self.fol_bc_ask()) + if len(bc_out) == 0: + self.valid = False + else: + self.valid = True + graph = bc_out[0][0][0] + s = bc_out[0][1] + while True: + new_graph = subst(s, graph) + if graph == new_graph: + break + graph = new_graph + self.make_table(graph) + self.context = None + self.draw_table() + + def fol_bc_ask(self): + """Run first-order backward chaining for the stored query against the knowledge + base, yielding proof trees together with their substitutions. + """ + KB = self.kb + query = self.query + + def fol_bc_or(KB, goal, theta): + for rule in KB.fetch_rules_for_goal(goal): + lhs, rhs = parse_definite_clause(standardize_variables(rule)) + for theta1 in fol_bc_and(KB, lhs, unify_mm(rhs, goal, theta)): + yield ([(goal, theta1[0])], theta1[1]) + + def fol_bc_and(KB, goals, theta): + if theta is None: + pass + elif not goals: + yield ([], theta) + else: + first, rest = goals[0], goals[1:] + for theta1 in fol_bc_or(KB, subst(theta, first), theta): + for theta2 in fol_bc_and(KB, rest, theta1[1]): + yield (theta1[0] + theta2[0], theta2[1]) + + return fol_bc_or(KB, query, {}) + + def make_table(self, graph): + """Lay out the proof tree as a table of node positions and edges (stored on the + instance) ready for rendering on the canvas. + """ + table = [] + pos = {} + links = set() + edges = set() + + def dfs(node, depth): + if len(table) <= depth: + table.append([]) + pos = len(table[depth]) + table[depth].append(node[0]) + for child in node[1]: + child_id = dfs(child, depth + 1) + links.add(((depth, pos), child_id)) + return (depth, pos) + + dfs(graph, 0) + y_off = 0.85 / len(table) + for i, row in enumerate(table): + x_off = 0.95 / len(row) + for j, node in enumerate(row): + pos[(i, j)] = (0.025 + j * x_off + (x_off - self.b) / 2, 0.025 + i * y_off + (y_off - self.l) / 2) + for p, c in links: + x1, y1 = pos[p] + x2, y2 = pos[c] + edges.add((x1 + self.b / 2, y1 + self.l, x2 + self.b / 2, y2)) + + self.table = table + self.pos = pos + self.edges = edges + + def mouse_click(self, x, y): + """Select the proof-tree node under the click position and redraw, showing its text.""" + x, y = x / self.width, y / self.height + for node in self.pos: + xs, ys = self.pos[node] + xe, ye = xs + self.b, ys + self.l + if xs <= x <= xe and ys <= y <= ye: + self.context = node + break + self.draw_table() + + def draw_table(self): + """Draw the proof-tree nodes and edges, plus a text area showing the content of + the currently selected node. + """ + self.clear() + self.strokeWidth(3) + self.stroke(0, 0, 0) + self.font("12px Arial") + if self.valid: + # draw nodes + for i, j in self.pos: + x, y = self.pos[(i, j)] + self.fill(200, 200, 200) + self.rect_n(x, y, self.b, self.l) + self.line_n(x, y, x + self.b, y) + self.line_n(x, y, x, y + self.l) + self.line_n(x + self.b, y, x + self.b, y + self.l) + self.line_n(x, y + self.l, x + self.b, y + self.l) + self.fill(0, 0, 0) + self.text_n(self.table[i][j], x + 0.01, y + self.l - 0.01) + # draw edges + for x1, y1, x2, y2 in self.edges: + self.line_n(x1, y1, x2, y2) + else: + self.fill(255, 0, 0) + self.rect_n(0, 0, 1, 1) + # text area + self.fill(255, 255, 255) + self.rect_n(0, 0.9, 1, 0.1) + self.strokeWidth(5) + self.stroke(0, 0, 0) + self.line_n(0, 0.9, 1, 0.9) + self.font("22px Arial") + self.fill(0, 0, 0) + self.text_n(self.table[self.context[0]][self.context[1]] if self.context else "Click for text", 0.025, 0.975) + self.update() + + +############################################################################################################ + +##################### Functions to assist plotting in search.ipynb #################### + +############################################################################################################ + + +def show_map(graph_data, node_colors=None): + """Draw a networkx graph of the given map data with coloured nodes, edge weight + labels and a legend describing the search-state colours. + """ + G = nx.Graph(graph_data['graph_dict']) + node_colors = node_colors or graph_data['node_colors'] + node_positions = graph_data['node_positions'] + node_label_pos = graph_data['node_label_positions'] + edge_weights = graph_data['edge_weights'] + + # set the size of the plot + plt.figure(figsize=(18, 13)) + # draw the graph (both nodes and edges) with locations from romania_locations + nx.draw(G, pos={k: node_positions[k] for k in G.nodes()}, + node_color=[node_colors[node] for node in G.nodes()], linewidths=0.3, edgecolors='k') + + # draw labels for nodes + node_label_handles = nx.draw_networkx_labels(G, pos=node_label_pos, font_size=14) + + # add a white bounding box behind the node labels + [label.set_bbox(dict(facecolor='white', edgecolor='none')) for label in node_label_handles.values()] + + # add edge labels to the graph + nx.draw_networkx_edge_labels(G, pos=node_positions, edge_labels=edge_weights, font_size=14) + + # add a legend + white_circle = lines.Line2D([], [], color="white", marker='o', markersize=15, markerfacecolor="white") + orange_circle = lines.Line2D([], [], color="orange", marker='o', markersize=15, markerfacecolor="orange") + red_circle = lines.Line2D([], [], color="red", marker='o', markersize=15, markerfacecolor="red") + gray_circle = lines.Line2D([], [], color="gray", marker='o', markersize=15, markerfacecolor="gray") + green_circle = lines.Line2D([], [], color="green", marker='o', markersize=15, markerfacecolor="green") + plt.legend((white_circle, orange_circle, red_circle, gray_circle, green_circle), + ('Un-explored', 'Frontier', 'Currently Exploring', 'Explored', 'Final Solution'), + numpoints=1, prop={'size': 16}, loc=(.8, .75)) + + # show the plot. No need to use in notebooks. nx.draw will show the graph itself. + plt.show() + + +# helper functions for visualisations + +def final_path_colors(initial_node_colors, problem, solution): + "Return a node_colors dict of the final path provided the problem and solution." + + # get initial node colors + final_colors = dict(initial_node_colors) + # color all the nodes in solution and starting node to green + final_colors[problem.initial] = "green" + for node in solution: + final_colors[node] = "green" + return final_colors + + +def display_visual(graph_data, user_input, algorithm=None, problem=None): + """Build interactive ipywidgets controls to animate a search algorithm on the map. + When ``user_input`` is True, also let the user choose the algorithm and the + start and goal cities; otherwise animate the supplied ``algorithm``. + """ + initial_node_colors = graph_data['node_colors'] + if user_input is False: + def slider_callback(iteration): + # don't show graph for the first time running the cell calling this function + try: + show_map(graph_data, node_colors=all_node_colors[iteration]) + except: + pass + + def visualize_callback(visualize): + if visualize is True: + button.value = False + + global all_node_colors + + iterations, all_node_colors, node = algorithm(problem) + solution = node.solution() + all_node_colors.append(final_path_colors(all_node_colors[0], problem, solution)) + + slider.max = len(all_node_colors) - 1 + + for i in range(slider.max + 1): + slider.value = i + # time.sleep(.5) + + slider = widgets.IntSlider(min=0, max=1, step=1, value=0) + slider_visual = widgets.interactive(slider_callback, iteration=slider) + display(slider_visual) + + button = widgets.ToggleButton(value=False) + button_visual = widgets.interactive(visualize_callback, visualize=button) + display(button_visual) + + if user_input is True: + node_colors = dict(initial_node_colors) + if isinstance(algorithm, dict): + assert set(algorithm.keys()).issubset({"Breadth First Tree Search", + "Depth First Tree Search", + "Breadth First Search", + "Depth First Graph Search", + "Best First Graph Search", + "Uniform Cost Search", + "Depth Limited Search", + "Iterative Deepening Search", + "Greedy Best First Search", + "A-star Search", + "Recursive Best First Search"}) + + algo_dropdown = widgets.Dropdown(description="Search algorithm: ", + options=sorted(list(algorithm.keys())), + value="Breadth First Tree Search") + display(algo_dropdown) + elif algorithm is None: + print("No algorithm to run.") + return 0 + + def slider_callback(iteration): + # don't show graph for the first time running the cell calling this function + try: + show_map(graph_data, node_colors=all_node_colors[iteration]) + except: + pass + + def visualize_callback(visualize): + if visualize is True: + button.value = False + + problem = GraphProblem(start_dropdown.value, end_dropdown.value, romania_map) + global all_node_colors + + user_algorithm = algorithm[algo_dropdown.value] + + iterations, all_node_colors, node = user_algorithm(problem) + solution = node.solution() + all_node_colors.append(final_path_colors(all_node_colors[0], problem, solution)) + + slider.max = len(all_node_colors) - 1 + + for i in range(slider.max + 1): + slider.value = i + # time.sleep(.5) + + start_dropdown = widgets.Dropdown(description="Start city: ", + options=sorted(list(node_colors.keys())), value="Arad") + display(start_dropdown) + + end_dropdown = widgets.Dropdown(description="Goal city: ", + options=sorted(list(node_colors.keys())), value="Fagaras") + display(end_dropdown) + + button = widgets.ToggleButton(value=False) + button_visual = widgets.interactive(visualize_callback, visualize=button) + display(button_visual) + + slider = widgets.IntSlider(min=0, max=1, step=1, value=0) + slider_visual = widgets.interactive(slider_callback, iteration=slider) + display(slider_visual) + + +# Function to plot NQueensCSP in csp.py and NQueensProblem in search.py +def plot_NQueens(solution): + """Draw an N-Queens chessboard with queen images for the given solution, which may + be a dict (from NQueensCSP) or a list (from NQueensProblem). + """ + n = len(solution) + board = np.array([2 * int((i + j) % 2) for j in range(n) for i in range(n)]).reshape((n, n)) + im = Image.open(os.path.join(_AIMA_ROOT, 'images/queen_s.png')) + height = im.size[1] + im = np.array(im).astype(float) / 255 + fig = plt.figure(figsize=(7, 7)) + ax = fig.add_subplot(111) + ax.set_title('{} Queens'.format(n)) + plt.imshow(board, cmap='binary', interpolation='nearest') + # NQueensCSP gives a solution as a dictionary + if isinstance(solution, dict): + for (k, v) in solution.items(): + newax = fig.add_axes([0.064 + (k * 0.112), 0.062 + ((7 - v) * 0.112), 0.1, 0.1], zorder=1) + newax.imshow(im) + newax.axis('off') + # NQueensProblem gives a solution as a list + elif isinstance(solution, list): + for (k, v) in enumerate(solution): + newax = fig.add_axes([0.064 + (k * 0.112), 0.062 + ((7 - v) * 0.112), 0.1, 0.1], zorder=1) + newax.imshow(im) + newax.axis('off') + fig.tight_layout() + plt.show() + + +# Function to plot a heatmap, given a grid +def heatmap(grid, cmap='binary', interpolation='nearest'): + """Display the given 2D grid as a heatmap.""" + fig = plt.figure(figsize=(7, 7)) + ax = fig.add_subplot(111) + ax.set_title('Heatmap') + plt.imshow(grid, cmap=cmap, interpolation=interpolation) + fig.tight_layout() + plt.show() + + +# Generates a gaussian kernel +def gaussian_kernel(l=5, sig=1.0): + """Return an ``l`` x ``l`` Gaussian kernel with standard deviation ``sig``.""" + ax = np.arange(-l // 2 + 1., l // 2 + 1.) + xx, yy = np.meshgrid(ax, ax) + kernel = np.exp(-(xx ** 2 + yy ** 2) / (2. * sig ** 2)) + return kernel + + +# Plots utility function for a POMDP +def plot_pomdp_utility(utility): + """Plot the piecewise-linear utility functions of a POMDP's actions and mark the + belief thresholds between the optimal actions. + """ + save = utility['0'][0] + delete = utility['1'][0] + ask_save = utility['2'][0] + ask_delete = utility['2'][-1] + left = (save[0] - ask_save[0]) / (save[0] - ask_save[0] + ask_save[1] - save[1]) + right = (delete[0] - ask_delete[0]) / (delete[0] - ask_delete[0] + ask_delete[1] - delete[1]) + + colors = ['g', 'b', 'k'] + for action in utility: + for value in utility[action]: + plt.plot(value, color=colors[int(action)]) + plt.vlines([left, right], -20, 10, linestyles='dashed', colors='c') + plt.ylim(-20, 13) + plt.xlim(0, 1) + plt.text(left / 2 - 0.05, 10, 'Save') + plt.text((right + left) / 2 - 0.02, 10, 'Ask') + plt.text((right + 1) / 2 - 0.07, 10, 'Delete') + plt.show() + + +def plot_model_boundary(dataset, attr1, attr2, model=None): + """Plot the decision boundary of a classifier over two attributes of a dataset. + Builds a mesh over the ``attr1``/``attr2`` plane, colours each cell by the + ``model``'s prediction, and overlays the training examples. + """ + # prepare data + examples = np.asarray(dataset.examples) + X = np.asarray([examples[:, attr1], examples[:, attr2]]) + y = examples[:, dataset.target] + h = 0.1 + + # create color maps + cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#00AAFF']) + cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#00AAFF']) + + # calculate min, max and limits + x_min, x_max = X[0].min() - 1, X[0].max() + 1 + y_min, y_max = X[1].min() - 1, X[1].max() + 1 + # mesh the grid + xx, yy = np.meshgrid(np.arange(x_min, x_max, h), + np.arange(y_min, y_max, h)) + Z = [] + for grid in zip(xx.ravel(), yy.ravel()): + # put them back to the example + grid = np.round(grid, decimals=1).tolist() + Z.append(model(grid)) + # Put the result into a color plot + Z = np.asarray(Z) + Z = Z.reshape(xx.shape) + plt.figure() + plt.pcolormesh(xx, yy, Z, cmap=cmap_light) + + # Plot also the training points + plt.scatter(X[0], X[1], c=y, cmap=cmap_bold) + plt.xlim(xx.min(), xx.max()) + plt.ylim(yy.min(), yy.max()) + plt.show() + + +# ______________________________________________________________________________ +# Visualizing search on a 2-D grid + + +def grid_search_steps(problem, strategy='astar'): + """Run a search on a :class:`~aima.search.GridProblem`, recording how it explores. + + Returns ``(explored, path)``: ``explored`` is the list of cells in the order + they are expanded (popped from the frontier), and ``path`` is the list of cells + from start to goal (``None`` if the goal is unreachable). ``strategy`` selects + the evaluation function -- ``'astar'`` (g + h), ``'ucs'`` (g), ``'bfs'`` (depth) + or ``'greedy'`` (h) -- which is what makes the different exploration patterns. + """ + evaluators = { + 'astar': lambda n: n.path_cost + problem.h(n), + 'ucs': lambda n: n.path_cost, + 'bfs': lambda n: n.depth, + 'greedy': lambda n: problem.h(n), + } + f = evaluators[strategy] + start = Node(problem.initial) + frontier = [(f(start), 0, start)] # (priority, tie-breaker, node) + best_cost = {problem.initial: start.path_cost} + explored, expanded = [], set() + counter = 1 + while frontier: + _, _, node = heapq.heappop(frontier) + if node.state in expanded: + continue + expanded.add(node.state) + explored.append(node.state) + if problem.goal_test(node.state): + return explored, [n.state for n in node.path()] + for child in node.expand(problem): + if child.state not in best_cost or child.path_cost < best_cost[child.state]: + best_cost[child.state] = child.path_cost + heapq.heappush(frontier, (f(child), counter, child)) + counter += 1 + return explored, None + + +def plot_grid_search(problem, explored, path=None, ax=None, title=None): + """Visualise a grid search: obstacles, the cells expanded (shaded by *when* they + were expanded), the solution path, and the start/goal. ``explored`` and ``path`` + are as returned by :func:`grid_search_steps`. Returns the matplotlib Axes. + """ + width, height = problem.width, problem.height + shade = np.full((height, width), np.nan) + for order, (x, y) in enumerate(explored): + shade[y, x] = order + + if ax is None: + _, ax = plt.subplots(figsize=(width * 0.45 + 1, height * 0.45 + 1)) + ax.imshow(shade, origin='lower', cmap='YlGnBu', + extent=(-0.5, width - 0.5, -0.5, height - 0.5)) + for (x, y) in problem.obstacles: + ax.add_patch(plt.Rectangle((x - 0.5, y - 0.5), 1, 1, color='0.2')) + if path: + ax.plot([c[0] for c in path], [c[1] for c in path], + color='orange', linewidth=2, marker='.', zorder=2) + (sx, sy), (gx, gy) = problem.initial, problem.goal + ax.scatter([sx], [sy], c='lime', s=130, marker='o', edgecolors='k', zorder=3) + ax.scatter([gx], [gy], c='red', s=160, marker='*', edgecolors='k', zorder=3) + ax.set_xticks(range(width)) + ax.set_yticks(range(height)) + ax.grid(True, color='0.85', linewidth=0.5) + ax.set_xlim(-0.5, width - 0.5) + ax.set_ylim(-0.5, height - 0.5) + ax.set_title(title or '{} cells expanded'.format(len(explored))) + return ax diff --git a/q-learning-9b/aima/perception.py b/q-learning-9b/aima/perception.py new file mode 100644 index 0000000..69a2a8a --- /dev/null +++ b/q-learning-9b/aima/perception.py @@ -0,0 +1,517 @@ +"""Perception (Chapter 24)""" + +import os + +import cv2 +import keras +import matplotlib.pyplot as plt +import numpy as np +import scipy.signal +from keras.datasets import mnist +from keras.layers import Dense, Activation, Flatten, InputLayer, Conv2D, MaxPooling2D +from keras.models import Sequential + +from aima.utils import gaussian_kernel_2D + +# repo root (the directory containing the `aima` package), so the bundled sample +# images resolve regardless of the current working directory +_AIMA_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# ____________________________________________________ +# 24.3 Early Image Processing Operators +# 24.3.1 Edge Detection + + +def array_normalization(array, range_min, range_max): + """Normalize an array in the range of (range_min, range_max)""" + if not isinstance(array, np.ndarray): + array = np.asarray(array) + array = array - np.min(array) + array = array * (range_max - range_min) / np.max(array) + range_min + return array + + +def gradient_edge_detector(image): + """ + Image edge detection by calculating gradients in the image + :param image: numpy ndarray or an iterable object + :return: numpy ndarray, representing a gray scale image + """ + if not isinstance(image, np.ndarray): + image = np.asarray(image) + # gradient filters of x and y direction edges + x_filter, y_filter = np.array([[1, -1]]), np.array([[1], [-1]]) + # convolution between filter and image to get edges + y_edges = scipy.signal.convolve2d(image, x_filter, 'same') + x_edges = scipy.signal.convolve2d(image, y_filter, 'same') + edges = array_normalization(x_edges + y_edges, 0, 255) + return edges + + +def gaussian_derivative_edge_detector(image): + """Image edge detector using derivative of gaussian kernels""" + if not isinstance(image, np.ndarray): + image = np.asarray(image) + gaussian_filter = gaussian_kernel_2D() + # init derivative of gaussian filters + x_filter = scipy.signal.convolve2d(gaussian_filter, np.asarray([[1, -1]]), 'same') + y_filter = scipy.signal.convolve2d(gaussian_filter, np.asarray([[1], [-1]]), 'same') + # extract edges using convolution + y_edges = scipy.signal.convolve2d(image, x_filter, 'same') + x_edges = scipy.signal.convolve2d(image, y_filter, 'same') + edges = array_normalization(x_edges + y_edges, 0, 255) + return edges + + +def hog(image, cell_size=8, bins=9, block_size=2): + """Histogram of Oriented Gradients (HOG) feature descriptor [#446]. + + Implements Dalal & Triggs (2005) -- the descriptor referenced by AIMA Chapter + 24 (Perception), for which the book gives no pseudocode. The steps are: + gradient magnitude and unsigned (0-180 deg) orientation; for each + ``cell_size`` x ``cell_size`` cell, a ``bins``-bin orientation histogram weighted + by gradient magnitude; L2 normalization over each ``block_size`` x ``block_size`` + block of cells; the normalized blocks concatenated into one feature vector. + + Source: N. Dalal & B. Triggs, "Histograms of Oriented Gradients for Human + Detection", CVPR 2005. + """ + image = np.asarray(image, dtype=float) + if image.ndim == 3: # collapse colour channels to grayscale + image = image.mean(axis=2) + + gy, gx = np.gradient(image) + magnitude = np.hypot(gx, gy) + orientation = np.rad2deg(np.arctan2(gy, gx)) % 180 # unsigned gradient angle + + cells_y, cells_x = image.shape[0] // cell_size, image.shape[1] // cell_size + bin_width = 180 / bins + histograms = np.zeros((cells_y, cells_x, bins)) + for cy in range(cells_y): + for cx in range(cells_x): + rows = slice(cy * cell_size, (cy + 1) * cell_size) + cols = slice(cx * cell_size, (cx + 1) * cell_size) + cell_bins = (orientation[rows, cols] // bin_width).astype(int) % bins + cell_mag = magnitude[rows, cols] + for k in range(bins): + histograms[cy, cx, k] = cell_mag[cell_bins == k].sum() + + eps = 1e-5 + features = [] + for by in range(cells_y - block_size + 1): + for bx in range(cells_x - block_size + 1): + block = histograms[by:by + block_size, bx:bx + block_size].ravel() + features.append(block / np.sqrt((block ** 2).sum() + eps ** 2)) + return np.concatenate(features) if features else histograms.ravel() + + +def laplacian_edge_detector(image): + """Extract image edge with laplacian filter""" + if not isinstance(image, np.ndarray): + image = np.asarray(image) + # init laplacian filter + laplacian_kernel = np.asarray([[0, -1, 0], [-1, 4, -1], [0, -1, 0]]) + # extract edges with convolution + edges = scipy.signal.convolve2d(image, laplacian_kernel, 'same') + edges = array_normalization(edges, 0, 255) + return edges + + +def show_edges(edges): + """ helper function to show edges picture""" + plt.imshow(edges, cmap='gray', vmin=0, vmax=255) + plt.axis('off') + plt.show() + + +# __________________________________________________ +# 24.3.3 Optical flow + + +def sum_squared_difference(pic1, pic2): + """SSD of two frames""" + pic1 = np.asarray(pic1) + pic2 = np.asarray(pic2) + assert pic1.shape == pic2.shape + min_ssd = np.inf + min_dxy = (np.inf, np.inf) + + # consider picture shift from -30 to 30 + for Dx in range(-30, 31): + for Dy in range(-30, 31): + # shift the image + shifted_pic = np.roll(pic2, Dx, axis=0) + shifted_pic = np.roll(shifted_pic, Dy, axis=1) + # calculate the difference + diff = np.sum((pic1 - shifted_pic) ** 2) + if diff < min_ssd: + min_dxy = (Dx, Dy) + min_ssd = diff + return min_dxy, min_ssd + + +# ____________________________________________________ +# segmentation + +def gen_gray_scale_picture(size, level=3): + """ + Generate a picture with different gray scale levels + + :param size: size of generated picture + :param level: the number of level of gray scales in the picture, + range (0, 255) are equally divided by number of levels + + :return image in numpy ndarray type + """ + assert level > 0 + # init an empty image + image = np.zeros((size, size)) + if level == 1: + return image + # draw a square on the left upper corner of the image + for x in range(size): + for y in range(size): + image[x, y] += (250 // (level - 1)) * (max(x, y) * level // size) + return image + + +gray_scale_image = gen_gray_scale_picture(3) + + +def probability_contour_detection(image, discs, threshold=0): + """ + Detect edges/contours by applying a set of discs to an image + :param image: an image in type of numpy ndarray + :param discs: a set of discs/filters to apply to pixels of image + :param threshold: threshold to tell whether the pixel at (x, y) is on an edge + :return image showing edges in numpy ndarray type + """ + # init an empty output image + res = np.zeros(image.shape) + step = discs[0].shape[0] + for x_i in range(0, image.shape[0] - step + 1, 1): + for y_i in range(0, image.shape[1] - step + 1, 1): + diff = [] + # apply each pair of discs and calculate the difference + for d in range(0, len(discs), 2): + disc1, disc2 = discs[d], discs[d + 1] + # crop the region of interest + region = image[x_i: x_i + step, y_i: y_i + step] + diff.append(np.sum(np.multiply(region, disc1)) - np.sum(np.multiply(region, disc2))) + if max(diff) > threshold: + # change color of the center of region + res[x_i + step // 2, y_i + step // 2] = 255 + return res + + +def group_contour_detection(image, cluster_num=2): + """ + Detecting contours in an image with k-means clustering + :param image: an image in numpy ndarray type + :param cluster_num: number of clusters in k-means + """ + img = image + Z = np.float32(img) + criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0) + K = cluster_num + # use kmeans in opencv-python + ret, label, center = cv2.kmeans(Z, K, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS) + center = np.uint8(center) + res = center[label.flatten()] + res2 = res.reshape(img.shape) + # show the image + # cv2.imshow('res2', res2) + # cv2.waitKey(0) + # cv2.destroyAllWindows() + return res2 + + +def image_to_graph(image): + """ + Convert an image to an graph in adjacent matrix form + """ + graph_dict = {} + for x in range(image.shape[0]): + for y in range(image.shape[1]): + graph_dict[(x, y)] = [(x + 1, y) if x + 1 < image.shape[0] else None, + (x, y + 1) if y + 1 < image.shape[1] else None] + return graph_dict + + +def generate_edge_weight(image, v1, v2): + """ + Find edge weight between two vertices in an image + :param image: image in numpy ndarray type + :param v1, v2: verticles in the image in form of (x index, y index) + """ + diff = abs(image[v1[0], v1[1]] - image[v2[0], v2[1]]) + return 255 - diff + + +class Graph: + """Graph in adjacent matrix to represent an image""" + + def __init__(self, image): + """image: ndarray""" + self.graph = image_to_graph(image) + # number of columns and rows + self.ROW = len(self.graph) + self.COL = 2 + self.image = image + # dictionary to save the maximum flow of each edge + self.flow = {} + # initialize the flow + for s in self.graph: + self.flow[s] = {} + for t in self.graph[s]: + if t: + self.flow[s][t] = generate_edge_weight(image, s, t) + + def bfs(self, s, t, parent): + """Breadth first search to tell whether there is an edge between source and sink + parent: a list to save the path between s and t""" + # queue to save the current searching frontier + queue = [s] + visited = [] + + while queue: + u = queue.pop(0) + for node in self.graph[u]: + # only select edge with positive flow + if node not in visited and node and self.flow[u][node] > 0: + queue.append(node) + visited.append(node) + parent.append((u, node)) + return True if t in visited else False + + def min_cut(self, source, sink): + """Find the minimum cut of the graph between source and sink""" + parent = [] + max_flow = 0 + + while self.bfs(source, sink, parent): + path_flow = np.inf + # find the minimum flow of s-t path + for s, t in parent: + path_flow = min(path_flow, self.flow[s][t]) + + max_flow += path_flow + + # update all edges between source and sink + for s in self.flow: + for t in self.flow[s]: + if t[0] <= sink[0] and t[1] <= sink[1]: + self.flow[s][t] -= path_flow + parent = [] + res = [] + for i in self.flow: + for j in self.flow[i]: + if self.flow[i][j] == 0 and generate_edge_weight(self.image, i, j) > 0: + res.append((i, j)) + return res + + +def gen_discs(init_scale, scales=1): + """ + Generate a collection of disc pairs by splitting an round discs with different angles + :param init_scale: the initial size of each half discs + :param scales: scale number of each type of half discs, the scale size will be doubled each time + :return: the collection of generated discs: [discs of scale1, discs of scale2...] + """ + discs = [] + for m in range(scales): + scale = init_scale * (m + 1) + disc = [] + # make the full empty dist + white = np.zeros((scale, scale)) + center = (scale - 1) / 2 + for i in range(scale): + for j in range(scale): + if (i - center) ** 2 + (j - center) ** 2 <= (center ** 2): + white[i, j] = 255 + # generate lower half and upper half + lower_half = np.copy(white) + lower_half[:(scale - 1) // 2, :] = 0 + upper_half = lower_half[::-1, ::-1] + # generate left half and right half + disc += [lower_half, upper_half, np.transpose(lower_half), np.transpose(upper_half)] + # generate upper-left, lower-right, upper-right, lower-left half discs + disc += [np.tril(white, 0), np.triu(white, 0), np.flip(np.tril(white, 0), axis=0), + np.flip(np.triu(white, 0), axis=0)] + discs.append(disc) + return discs + + +# __________________________________________________ +# 24.4 Classifying Images + + +def load_MINST(train_size, val_size, test_size): + """Load MINST dataset from keras""" + (x_train, y_train), (x_test, y_test) = mnist.load_data() + total_size = len(x_train) + if train_size + val_size > total_size: + train_size = total_size - val_size + x_train = x_train.reshape(x_train.shape[0], 1, 28, 28) + x_test = x_test.reshape(x_test.shape[0], 1, 28, 28) + x_train = x_train.astype('float32') + x_train /= 255 + test_x = x_test.astype('float32') + test_x /= 255 + y_train = keras.utils.to_categorical(y_train, 10) + y_test = keras.utils.to_categorical(y_test, 10) + return ((x_train[:train_size], y_train[:train_size]), + (x_train[train_size:train_size + val_size], y_train[train_size:train_size + val_size]), + (x_test[:test_size], y_test[:test_size])) + + +def simple_convnet(size=3, num_classes=10): + """ + Simple convolutional network for digit recognition + :param size: number of convolution layers + :param num_classes: number of output classes + :return a convolution network in keras model type + """ + model = Sequential() + # add input layer for images of size (28, 28) + model.add(InputLayer(input_shape=(1, 28, 28))) + # add convolution layers and max pooling layers + for _ in range(size): + model.add(Conv2D(32, (2, 2), padding='same', kernel_initializer='random_uniform')) + model.add(MaxPooling2D(padding='same')) + + # add flatten layer and output layers + model.add(Flatten()) + model.add(Dense(num_classes)) + model.add(Activation('softmax')) + + # compile model + model.compile(loss='categorical_crossentropy', + metrics=['accuracy']) + print(model.summary()) + return model + + +def train_model(model): + """Train the simple convolution network""" + # load dataset + (train_x, train_y), (val_x, val_y), (test_x, test_y) = load_MINST(1000, 100, 100) + model.fit(train_x, train_y, validation_data=(val_x, val_y), epochs=5, verbose=2, batch_size=32) + scores = model.evaluate(test_x, test_y, verbose=1) + print(scores) + return model + + +# _____________________________________________________ +# 24.5 DETECTING OBJECTS + + +def selective_search(image): + """ + Selective search for object detection + :param image: str, the path of image or image in ndarray type with 3 channels + :return list of bounding boxes, each element is in form of [x_min, y_min, x_max, y_max] + """ + if not image: + im = cv2.imread(os.path.join(_AIMA_ROOT, "images/stapler1-test.png")) + elif isinstance(image, str): + im = cv2.imread(image) + else: + im = np.stack(image * 3, axis=-1) + + # use opencv python to extract bounding box with selective search + ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation() + ss.setBaseImage(im) + ss.switchToSelectiveSearchQuality() + rects = ss.process() + + # show bounding boxes with the input image + image_out = im.copy() + for rect in rects[:100]: + print(rect) + x, y, w, h = rect + cv2.rectangle(image_out, (x, y), (x + w, y + h), (0, 255, 0), 1, cv2.LINE_AA) + cv2.imshow("Output", image_out) + cv2.waitKey(0) + return rects + + +# faster RCNN +def pool_rois(feature_map, rois, pooled_height, pooled_width): + """ + Applies ROI pooling for a single image and various ROIs + :param feature_map: ndarray, in shape of (width, height, channel) + :param rois: list of roi + :param pooled_height: height of pooled area + :param pooled_width: width of pooled area + :return list of pooled features + """ + + def curried_pool_roi(roi): + return pool_roi(feature_map, roi, pooled_height, pooled_width) + + pooled_areas = list(map(curried_pool_roi, rois)) + return pooled_areas + + +def pool_roi(feature_map, roi, pooled_height, pooled_width): + """ + Applies a single ROI pooling to a single image + :param feature_map: ndarray, in shape of (width, height, channel) + :param roi: region of interest, in form of [x_min_ratio, y_min_ratio, x_max_ratio, y_max_ratio] + :return feature of pooling output, in shape of (pooled_width, pooled_height) + """ + + # Compute the region of interest + feature_map_height = int(feature_map.shape[0]) + feature_map_width = int(feature_map.shape[1]) + + h_start = int(feature_map_height * roi[0]) + w_start = int(feature_map_width * roi[1]) + h_end = int(feature_map_height * roi[2]) + w_end = int(feature_map_width * roi[3]) + + region = feature_map[h_start:h_end, w_start:w_end, :] + + # Divide the region into non overlapping areas + region_height = h_end - h_start + region_width = w_end - w_start + h_step = region_height // pooled_height + w_step = region_width // pooled_width + + areas = [[( + i * h_step, + j * w_step, + (i + 1) * h_step if i + 1 < pooled_height else region_height, + (j + 1) * w_step if j + 1 < pooled_width else region_width) + for j in range(pooled_width)] + for i in range(pooled_height)] + + # take the maximum of each area and stack the result + def pool_area(x): + return np.max(region[x[0]:x[2], x[1]:x[3], :]) + + pooled_features = np.stack([[pool_area(x) for x in row] for row in areas]) + return pooled_features + +# faster rcnn demo can be installed and shown in jupyter notebook +# def faster_rcnn_demo(directory): +# """ +# show the demo of rcnn, the model is from +# @inproceedings{renNIPS15fasterrcnn, +# Author = {Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun}, +# Title = {Faster {R-CNN}: Towards Real-Time Object Detection +# with Region Proposal Networks}, +# Booktitle = {Advances in Neural Information Processing Systems ({NIPS})}, +# Year = {2015}} +# :param directory: the directory where the faster rcnn model is installed +# """ +# os.chdir(directory + '/lib') +# # make file +# os.system("make clean") +# os.system("make") +# # run demo +# os.chdir(directory) +# os.system("./tools/demo.py") +# return 0 diff --git a/q-learning-9b/aima/planning.py b/q-learning-9b/aima/planning.py new file mode 100644 index 0000000..4a14549 --- /dev/null +++ b/q-learning-9b/aima/planning.py @@ -0,0 +1,2524 @@ +"""Planning (Chapters 10-11)""" + +import copy +import itertools +from collections import deque, defaultdict +from functools import reduce as _reduce + +import numpy as np + +from aima import search +from aima.csp import sat_up, NaryCSP, Constraint, ac_search_solver, is_constraint +from aima.logic import FolKB, conjuncts, unify_mm, associate, SAT_plan, cdcl_satisfiable +from aima.search import Node +from aima.utils import Expr, expr, first + + +class PlanningProblem: + """ + Planning Domain Definition Language (PlanningProblem) used to define a search problem. + It stores states in a knowledge base consisting of first order logic statements. + The conjunction of these logical statements completely defines a state. + """ + + def __init__(self, initial, goals, actions, domain=None): + self.initial = self.convert(initial) if domain is None else self.convert(initial) + self.convert(domain) + self.goals = self.convert(goals) + self.actions = actions + self.domain = domain + + def convert(self, clauses): + """Converts strings into exprs""" + if not isinstance(clauses, Expr): + if len(clauses) > 0: + clauses = expr(clauses) + else: + clauses = [] + try: + clauses = conjuncts(clauses) + except AttributeError: + pass + + new_clauses = [] + for clause in clauses: + if clause.op == '~': + new_clauses.append(expr('Not' + str(clause.args[0]))) + else: + new_clauses.append(clause) + return new_clauses + + def expand_fluents(self, name=None): + """ + Generate all ground fluents obtainable by binding the problem's objects to the + predicates appearing in the initial state, goals and action effects. If a fluent + ``name`` is given, only expansions of that single fluent are produced. When a domain + is defined, candidate ground fluents are filtered through a FolKB so that only those + consistent with the domain are kept. + """ + + kb = None + if self.domain: + kb = FolKB(self.convert(self.domain)) + for action in self.actions: + if action.precond: + for fests in set(action.precond).union(action.effect).difference(self.convert(action.domain)): + if fests.op[:3] != 'Not': + kb.tell(expr(str(action.domain) + ' ==> ' + str(fests))) + + objects = set(arg for clause in set(self.initial + self.goals) for arg in clause.args) + fluent_list = [] + if name is not None: + for fluent in self.initial + self.goals: + if str(fluent) == name: + fluent_list.append(fluent) + break + else: + fluent_list = list(map(lambda fluent: Expr(fluent[0], *fluent[1]), + {fluent.op: fluent.args for fluent in self.initial + self.goals + + [clause for action in self.actions for clause in action.effect if + clause.op[:3] != 'Not']}.items())) + + expansions = [] + for fluent in fluent_list: + for permutation in itertools.permutations(objects, len(fluent.args)): + new_fluent = Expr(fluent.op, *permutation) + if (self.domain and kb.ask(new_fluent) is not False) or not self.domain: + expansions.append(new_fluent) + + return expansions + + def expand_actions(self, name=None): + """Generate all possible actions with variable bindings for precondition selection heuristic""" + + has_domains = all(action.domain for action in self.actions if action.precond) + kb = None + if has_domains: + kb = FolKB(self.initial) + for action in self.actions: + if action.precond: + kb.tell(expr(str(action.domain) + ' ==> ' + str(action))) + + objects = set(arg for clause in self.initial for arg in clause.args) + expansions = [] + action_list = [] + if name is not None: + for action in self.actions: + if str(action.name) == name: + action_list.append(action) + break + else: + action_list = self.actions + + for action in action_list: + for permutation in itertools.permutations(objects, len(action.args)): + bindings = unify_mm(Expr(action.name, *action.args), Expr(action.name, *permutation)) + if bindings is not None: + new_args = [] + for arg in action.args: + if arg in bindings: + new_args.append(bindings[arg]) + else: + new_args.append(arg) + new_expr = Expr(str(action.name), *new_args) + if (has_domains and kb.ask(new_expr) is not False) or ( + has_domains and not action.precond) or not has_domains: + new_preconds = [] + for precond in action.precond: + new_precond_args = [] + for arg in precond.args: + if arg in bindings: + new_precond_args.append(bindings[arg]) + else: + new_precond_args.append(arg) + new_precond = Expr(str(precond.op), *new_precond_args) + new_preconds.append(new_precond) + new_effects = [] + for effect in action.effect: + new_effect_args = [] + for arg in effect.args: + if arg in bindings: + new_effect_args.append(bindings[arg]) + else: + new_effect_args.append(arg) + new_effect = Expr(str(effect.op), *new_effect_args) + new_effects.append(new_effect) + expansions.append(Action(new_expr, new_preconds, new_effects)) + + return expansions + + def is_strips(self): + """ + Returns True if the problem does not contain negative literals in preconditions and goals + """ + return (all(clause.op[:3] != 'Not' for clause in self.goals) and + all(clause.op[:3] != 'Not' for action in self.actions for clause in action.precond)) + + def goal_test(self): + """Checks if the goals have been reached""" + return all(goal in self.initial for goal in self.goals) + + def act(self, action): + """ + Performs the action given as argument. + Note that action is an Expr like expr('Remove(Glass, Table)') or expr('Eat(Sandwich)') + """ + action_name = action.op + args = action.args + list_action = first(a for a in self.actions if a.name == action_name) + if list_action is None: + raise Exception("Action '{}' not found".format(action_name)) + if not list_action.check_precond(self.initial, args): + raise Exception("Action '{}' pre-conditions not satisfied".format(action)) + self.initial = list_action(self.initial, args).clauses + + +class Action: + """ + Defines an action schema using preconditions and effects. + Use this to describe actions in PlanningProblem. + action is an Expr where variables are given as arguments(args). + Precondition and effect are both lists with positive and negative literals. + Negative preconditions and effects are defined by adding a 'Not' before the name of the clause + Example: + precond = [expr("Human(person)"), expr("Hungry(Person)"), expr("NotEaten(food)")] + effect = [expr("Eaten(food)"), expr("Hungry(person)")] + eat = Action(expr("Eat(person, food)"), precond, effect) + """ + + def __init__(self, action, precond, effect, domain=None): + if isinstance(action, str): + action = expr(action) + self.name = action.op + self.args = action.args + self.precond = self.convert(precond) if domain is None else self.convert(precond) + self.convert(domain) + self.effect = self.convert(effect) + self.domain = domain + + def __call__(self, kb, args): + return self.act(kb, args) + + def __repr__(self): + return '{}'.format(Expr(self.name, *self.args)) + + def convert(self, clauses): + """Converts strings into Exprs""" + if isinstance(clauses, Expr): + clauses = conjuncts(clauses) + for i in range(len(clauses)): + if clauses[i].op == '~': + clauses[i] = expr('Not' + str(clauses[i].args[0])) + + elif isinstance(clauses, str): + clauses = clauses.replace('~', 'Not') + if len(clauses) > 0: + clauses = expr(clauses) + + try: + clauses = conjuncts(clauses) + except AttributeError: + pass + + return clauses + + def relaxed(self): + """ + Removes delete list from the action by removing all negative literals from action's effect + """ + return Action(Expr(self.name, *self.args), self.precond, + list(filter(lambda effect: effect.op[:3] != 'Not', self.effect))) + + def substitute(self, e, args): + """Replaces variables in expression with their respective Propositional symbol""" + + new_args = list(e.args) + for num, x in enumerate(e.args): + for i, _ in enumerate(self.args): + if self.args[i] == x: + new_args[num] = args[i] + return Expr(e.op, *new_args) + + def check_precond(self, kb, args): + """Checks if the precondition is satisfied in the current state""" + + if isinstance(kb, list): + kb = FolKB(kb) + for clause in self.precond: + if self.substitute(clause, args) not in kb.clauses: + return False + return True + + def act(self, kb, args): + """Executes the action on the state's knowledge base""" + + if isinstance(kb, list): + kb = FolKB(kb) + + if not self.check_precond(kb, args): + raise Exception('Action pre-conditions not satisfied') + for clause in self.effect: + kb.tell(self.substitute(clause, args)) + if clause.op[:3] == 'Not': + new_clause = Expr(clause.op[3:], *clause.args) + + if kb.ask(self.substitute(new_clause, args)) is not False: + kb.retract(self.substitute(new_clause, args)) + else: + new_clause = Expr('Not' + clause.op, *clause.args) + + if kb.ask(self.substitute(new_clause, args)) is not False: + kb.retract(self.substitute(new_clause, args)) + + return kb + + +def goal_test(goals, state): + """Generic goal testing helper function""" + + if isinstance(state, list): + kb = FolKB(state) + else: + kb = state + return all(kb.ask(q) is not False for q in goals) + + +def air_cargo(): + """ + [Figure 10.1] AIR-CARGO-PROBLEM + + An air-cargo shipment problem for delivering cargo to different locations, + given the starting location and airplanes. + + Example: + >>> from planning import * + >>> ac = air_cargo() + >>> ac.goal_test() + False + >>> ac.act(expr('Load(C2, P2, JFK)')) + >>> ac.act(expr('Load(C1, P1, SFO)')) + >>> ac.act(expr('Fly(P1, SFO, JFK)')) + >>> ac.act(expr('Fly(P2, JFK, SFO)')) + >>> ac.act(expr('Unload(C2, P2, SFO)')) + >>> ac.goal_test() + False + >>> ac.act(expr('Unload(C1, P1, JFK)')) + >>> ac.goal_test() + True + >>> + """ + + return PlanningProblem(initial='At(C1, SFO) & At(C2, JFK) & At(P1, SFO) & At(P2, JFK)', + goals='At(C1, JFK) & At(C2, SFO)', + actions=[Action('Load(c, p, a)', + precond='At(c, a) & At(p, a)', + effect='In(c, p) & ~At(c, a)', + domain='Cargo(c) & Plane(p) & Airport(a)'), + Action('Unload(c, p, a)', + precond='In(c, p) & At(p, a)', + effect='At(c, a) & ~In(c, p)', + domain='Cargo(c) & Plane(p) & Airport(a)'), + Action('Fly(p, f, to)', + precond='At(p, f)', + effect='At(p, to) & ~At(p, f)', + domain='Plane(p) & Airport(f) & Airport(to)')], + domain='Cargo(C1) & Cargo(C2) & Plane(P1) & Plane(P2) & Airport(SFO) & Airport(JFK)') + + +def spare_tire(): + """ + [Figure 10.2] SPARE-TIRE-PROBLEM + + A problem involving changing the flat tire of a car + with a spare tire from the trunk. + + Example: + >>> from planning import * + >>> st = spare_tire() + >>> st.goal_test() + False + >>> st.act(expr('Remove(Spare, Trunk)')) + >>> st.act(expr('Remove(Flat, Axle)')) + >>> st.goal_test() + False + >>> st.act(expr('PutOn(Spare, Axle)')) + >>> st.goal_test() + True + >>> + """ + + return PlanningProblem(initial='At(Flat, Axle) & At(Spare, Trunk)', + goals='At(Spare, Axle) & At(Flat, Ground)', + actions=[Action('Remove(obj, loc)', + precond='At(obj, loc)', + effect='At(obj, Ground) & ~At(obj, loc)', + domain='Tire(obj)'), + Action('PutOn(t, Axle)', + precond='At(t, Ground) & ~At(Flat, Axle)', + effect='At(t, Axle) & ~At(t, Ground)', + domain='Tire(t)'), + Action('LeaveOvernight', + precond='', + effect='~At(Spare, Ground) & ~At(Spare, Axle) & ~At(Spare, Trunk) & \ + ~At(Flat, Ground) & ~At(Flat, Axle) & ~At(Flat, Trunk)')], + domain='Tire(Flat) & Tire(Spare)') + + +def three_block_tower(): + """ + [Figure 10.3] THREE-BLOCK-TOWER + + A blocks-world problem of stacking three blocks in a certain configuration, + also known as the Sussman Anomaly. + + Example: + >>> from planning import * + >>> tbt = three_block_tower() + >>> tbt.goal_test() + False + >>> tbt.act(expr('MoveToTable(C, A)')) + >>> tbt.act(expr('Move(B, Table, C)')) + >>> tbt.goal_test() + False + >>> tbt.act(expr('Move(A, Table, B)')) + >>> tbt.goal_test() + True + >>> + """ + return PlanningProblem(initial='On(A, Table) & On(B, Table) & On(C, A) & Clear(B) & Clear(C)', + goals='On(A, B) & On(B, C)', + actions=[Action('Move(b, x, y)', + precond='On(b, x) & Clear(b) & Clear(y)', + effect='On(b, y) & Clear(x) & ~On(b, x) & ~Clear(y)', + domain='Block(b) & Block(y)'), + Action('MoveToTable(b, x)', + precond='On(b, x) & Clear(b)', + effect='On(b, Table) & Clear(x) & ~On(b, x)', + domain='Block(b) & Block(x)')], + domain='Block(A) & Block(B) & Block(C)') + + +def simple_blocks_world(): + """ + SIMPLE-BLOCKS-WORLD + + A simplified definition of the Sussman Anomaly problem. + + Example: + >>> from planning import * + >>> sbw = simple_blocks_world() + >>> sbw.goal_test() + False + >>> sbw.act(expr('ToTable(A, B)')) + >>> sbw.act(expr('FromTable(B, A)')) + >>> sbw.goal_test() + False + >>> sbw.act(expr('FromTable(C, B)')) + >>> sbw.goal_test() + True + >>> + """ + + return PlanningProblem(initial='On(A, B) & Clear(A) & OnTable(B) & OnTable(C) & Clear(C)', + goals='On(B, A) & On(C, B)', + actions=[Action('ToTable(x, y)', + precond='On(x, y) & Clear(x)', + effect='~On(x, y) & Clear(y) & OnTable(x)'), + Action('FromTable(y, x)', + precond='OnTable(y) & Clear(y) & Clear(x)', + effect='~OnTable(y) & ~Clear(x) & On(y, x)')]) + + +def have_cake_and_eat_cake_too(): + """ + [Figure 10.7] CAKE-PROBLEM + + A problem where we begin with a cake and want to + reach the state of having a cake and having eaten a cake. + The possible actions include baking a cake and eating a cake. + + Example: + >>> from planning import * + >>> cp = have_cake_and_eat_cake_too() + >>> cp.goal_test() + False + >>> cp.act(expr('Eat(Cake)')) + >>> cp.goal_test() + False + >>> cp.act(expr('Bake(Cake)')) + >>> cp.goal_test() + True + >>> + """ + + return PlanningProblem(initial='Have(Cake)', + goals='Have(Cake) & Eaten(Cake)', + actions=[Action('Eat(Cake)', + precond='Have(Cake)', + effect='Eaten(Cake) & ~Have(Cake)'), + Action('Bake(Cake)', + precond='~Have(Cake)', + effect='Have(Cake)')]) + + +def shopping_problem(): + """ + SHOPPING-PROBLEM + + A problem of acquiring some items given their availability at certain stores. + + Example: + >>> from planning import * + >>> sp = shopping_problem() + >>> sp.goal_test() + False + >>> sp.act(expr('Go(Home, HW)')) + >>> sp.act(expr('Buy(Drill, HW)')) + >>> sp.act(expr('Go(HW, SM)')) + >>> sp.act(expr('Buy(Banana, SM)')) + >>> sp.goal_test() + False + >>> sp.act(expr('Buy(Milk, SM)')) + >>> sp.goal_test() + True + >>> + """ + + return PlanningProblem(initial='At(Home) & Sells(SM, Milk) & Sells(SM, Banana) & Sells(HW, Drill)', + goals='Have(Milk) & Have(Banana) & Have(Drill)', + actions=[Action('Buy(x, store)', + precond='At(store) & Sells(store, x)', + effect='Have(x)', + domain='Store(store) & Item(x)'), + Action('Go(x, y)', + precond='At(x)', + effect='At(y) & ~At(x)', + domain='Place(x) & Place(y)')], + domain='Place(Home) & Place(SM) & Place(HW) & Store(SM) & Store(HW) & ' + 'Item(Milk) & Item(Banana) & Item(Drill)') + + +def socks_and_shoes(): + """ + SOCKS-AND-SHOES-PROBLEM + + A task of wearing socks and shoes on both feet + + Example: + >>> from planning import * + >>> ss = socks_and_shoes() + >>> ss.goal_test() + False + >>> ss.act(expr('RightSock')) + >>> ss.act(expr('RightShoe')) + >>> ss.act(expr('LeftSock')) + >>> ss.goal_test() + False + >>> ss.act(expr('LeftShoe')) + >>> ss.goal_test() + True + >>> + """ + + return PlanningProblem(initial='', + goals='RightShoeOn & LeftShoeOn', + actions=[Action('RightShoe', + precond='RightSockOn', + effect='RightShoeOn'), + Action('RightSock', + precond='', + effect='RightSockOn'), + Action('LeftShoe', + precond='LeftSockOn', + effect='LeftShoeOn'), + Action('LeftSock', + precond='', + effect='LeftSockOn')]) + + +def logistics_problem(initial_state=None, goal_state=None): + """ + LOGISTICS-PROBLEM + + A logistics problem where a robot moves between places, picking up and + putting down containers in order to deliver them to their destinations. + + Example: + >>> from planning import * + >>> lp = logistics_problem(goal_state='In(C2, D3) & In(C3, D3)') + >>> lp.goal_test() + False + >>> lp.act(expr('PutDown(R1, C1, D1)')) + >>> lp.act(expr('PickUp(R1, C2, D1)')) + >>> lp.act(expr('Move(R1, D1, D3)')) + >>> lp.act(expr('PutDown(R1, C2, D3)')) + >>> lp.act(expr('Move(R1, D3, D2)')) + >>> lp.act(expr('PickUp(R1, C3, D2)')) + >>> lp.act(expr('Move(R1, D2, D3)')) + >>> lp.goal_test() + False + >>> lp.act(expr('PutDown(R1, C3, D3)')) + >>> lp.goal_test() + True + >>> + """ + if initial_state is None: + initial_state = 'In(C1, R1) & In(C2, D1) & In(C3, D2) & In(R1, D1) & Holding(R1)' + if goal_state is None: + raise ValueError('Goal must be defined') + + return PlanningProblem(initial=initial_state, + goals=goal_state, + actions=[Action('PickUp(r, c, d)', + precond='In(r, d) & In(c, d) & ~Holding(r)', + effect='Holding(r) & ~In(c, d) & In(c, r)', + domain='Robot(r) & Place(d) & Container(c)'), + Action('PutDown(r, c, d)', + precond='In(r, d) & In(c, r) & Holding(r)', + effect='~Holding(r) & ~In(c, r) & In(c, d)', + domain='Robot(r) & Place(d) & Container(c)'), + Action('Move(r, d_start, d_end)', + precond='In(r, d_start)', + effect='~In(r, d_start) & In(r, d_end)', + domain='Robot(r) & Place(d_start) & Place(d_end)')], + domain='Container(C1) & Container(C2) & Container(C3) & ' + 'Place(D1) & Place(D2) & Place(D3) & Robot(R1)') + + +def blocks_world(initial, goals, blocks): + """ + GENERALIZED-BLOCKS-WORLD-PROBLEM + + A flexible constructor for creating blocks-world planning problems. + Any initial and goal configuration can be specified for a given set of blocks. + + Example: + >>> from planning import * + >>> initial_state = 'On(C, A) & On(A, Table) & On(B, Table) & Clear(C) & Clear(B)' + >>> goal_state = 'On(A, B) & On(B, C)' + >>> sussman_anomaly = blocks_world(initial_state, goal_state, ['A', 'B', 'C']) + >>> sussman_anomaly.goal_test() + False + >>> sussman_anomaly.act(expr('MoveToTable(C, A)')) + >>> sussman_anomaly.act(expr('Move(B, Table, C)')) + >>> sussman_anomaly.act(expr('Move(A, Table, B)')) + >>> sussman_anomaly.goal_test() + True + >>> + """ + # Dynamically generate the domain knowledge based on the list of blocks + domain = ' & '.join('Block({})'.format(b) for b in blocks) + + actions = [Action('Move(b, x, y)', + precond='On(b, x) & Clear(b) & Clear(y)', + effect='On(b, y) & Clear(x) & ~On(b, x) & ~Clear(y)', + domain='Block(b) & Block(y)'), + Action('MoveToTable(b, x)', + precond='On(b, x) & Clear(b)', + effect='On(b, Table) & Clear(x) & ~On(b, x)', + domain='Block(b) & Block(x)')] + + return PlanningProblem(initial=initial, + goals=goals, + actions=actions, + domain=domain) + + +def rush_hour(): + """ + RUSH-HOUR-PROBLEM (non-numeric version) + + A planning problem for the Rush Hour sliding block puzzle. The goal is to + maneuver the RedCar to the exit. This version uses non-numeric symbols for + grid positions (e.g. R1, C1) instead of integers. + + This specific instance uses: + - RedCar (2x1, horizontal) starting at (R3, C1) + - GreenTruck (3x1, vertical) starting at (R1, C4) + - BlueCar (2x1, vertical) starting at (R5, C2) + """ + initial_state = 'At(RedCar, R3, C1) & At(GreenTruck, R1, C4) & At(BlueCar, R5, C2) & ' \ + 'IsHorizontal(RedCar) & IsVertical(GreenTruck) & IsVertical(BlueCar) & ' \ + 'Clear(R1, C1) & Clear(R1, C2) & Clear(R1, C3) & Clear(R1, C5) & Clear(R1, C6) & ' \ + 'Clear(R2, C1) & Clear(R2, C2) & Clear(R2, C3) & Clear(R2, C5) & Clear(R2, C6) & ' \ + 'Clear(R3, C3) & Clear(R3, C4) & Clear(R3, C5) & Clear(R3, C6) & ' \ + 'Clear(R4, C1) & Clear(R4, C2) & Clear(R4, C3) & Clear(R4, C4) & Clear(R4, C5) & Clear(R4, C6) & ' \ + 'Clear(R5, C1) & Clear(R5, C3) & Clear(R5, C4) & Clear(R5, C5) & Clear(R5, C6) & ' \ + 'Clear(R6, C1) & Clear(R6, C3) & Clear(R6, C4) & Clear(R6, C5) & Clear(R6, C6)' + + # Goal state: the RedCar's left-most part is at column C5. + goal_state = 'At(RedCar, R3, C5)' + + domain = 'Vehicle(RedCar) & Vehicle(GreenTruck) & Vehicle(BlueCar) & ' \ + 'Car(RedCar) & Truck(GreenTruck) & Car(BlueCar) & ' \ + 'Row(R1) & Row(R2) & Row(R3) & Row(R4) & Row(R5) & Row(R6) & ' \ + 'Col(C1) & Col(C2) & Col(C3) & Col(C4) & Col(C5) & Col(C6) & ' \ + 'NextTo(R1, R2) & NextTo(R2, R3) & NextTo(R3, R4) & NextTo(R4, R5) & NextTo(R5, R6) & ' \ + 'NextTo(C1, C2) & NextTo(C2, C3) & NextTo(C3, C4) & NextTo(C4, C5) & NextTo(C5, C6)' + + actions = [ + # car actions (length 2) + Action('MoveRightCar(v, r, c1, c2, c3)', + precond='At(v, r, c1) & Car(v) & IsHorizontal(v) & NextTo(c1, c2) & NextTo(c2, c3) & Clear(r, c3)', + effect='At(v, r, c2) & ~At(v, r, c1) & Clear(r, c1) & ~Clear(r, c3)', + domain='Vehicle(v) & Row(r) & Col(c1) & Col(c2) & Col(c3)'), + Action('MoveLeftCar(v, r, c1, c2, c3)', + precond='At(v, r, c2) & Car(v) & IsHorizontal(v) & NextTo(c1, c2) & NextTo(c2, c3) & Clear(r, c1)', + effect='At(v, r, c1) & ~At(v, r, c2) & Clear(r, c3) & ~Clear(r, c1)', + domain='Vehicle(v) & Row(r) & Col(c1) & Col(c2) & Col(c3)'), + Action('MoveDownCar(v, r1, r2, r3, c)', + precond='At(v, r1, c) & Car(v) & IsVertical(v) & NextTo(r1, r2) & NextTo(r2, r3) & Clear(r3, c)', + effect='At(v, r2, c) & ~At(v, r1, c) & Clear(r1, c) & ~Clear(r3, c)', + domain='Vehicle(v) & Row(r1) & Row(r2) & Row(r3) & Col(c)'), + Action('MoveUpCar(v, r1, r2, r3, c)', + precond='At(v, r2, c) & Car(v) & IsVertical(v) & NextTo(r1, r2) & NextTo(r2, r3) & Clear(r1, c)', + effect='At(v, r1, c) & ~At(v, r2, c) & Clear(r3, c) & ~Clear(r1, c)', + domain='Vehicle(v) & Row(r1) & Row(r2) & Row(r3) & Col(c)'), + # truck actions (length 3) + Action('MoveRightTruck(v, r, c1, c2, c3, c4)', + precond='At(v, r, c1) & Truck(v) & IsHorizontal(v) & NextTo(c1, c2) & NextTo(c2, c3) & ' + 'NextTo(c3, c4) & Clear(r, c4)', + effect='At(v, r, c2) & ~At(v, r, c1) & Clear(r, c1) & ~Clear(r, c4)', + domain='Vehicle(v) & Row(r) & Col(c1) & Col(c2) & Col(c3) & Col(c4)'), + Action('MoveLeftTruck(v, r, c1, c2, c3, c4)', + precond='At(v, r, c2) & Truck(v) & IsHorizontal(v) & NextTo(c1, c2) & NextTo(c2, c3) & ' + 'NextTo(c3, c4) & Clear(r, c1)', + effect='At(v, r, c1) & ~At(v, r, c2) & Clear(r, c4) & ~Clear(r, c1)', + domain='Vehicle(v) & Row(r) & Col(c1) & Col(c2) & Col(c3) & Col(c4)'), + Action('MoveDownTruck(v, r1, r2, r3, r4, c)', + precond='At(v, r1, c) & Truck(v) & IsVertical(v) & NextTo(r1, r2) & NextTo(r2, r3) & ' + 'NextTo(r3, r4) & Clear(r4, c)', + effect='At(v, r2, c) & ~At(v, r1, c) & Clear(r1, c) & ~Clear(r4, c)', + domain='Vehicle(v) & Row(r1) & Row(r2) & Row(r3) & Row(r4) & Col(c)'), + Action('MoveUpTruck(v, r1, r2, r3, r4, c)', + precond='At(v, r2, c) & Truck(v) & IsVertical(v) & NextTo(r1, r2) & NextTo(r2, r3) & ' + 'NextTo(r3, r4) & Clear(r1, c)', + effect='At(v, r1, c) & ~At(v, r2, c) & Clear(r4, c) & ~Clear(r1, c)', + domain='Vehicle(v) & Row(r1) & Row(r2) & Row(r3) & Row(r4) & Col(c)')] + + return PlanningProblem(initial=initial_state, + goals=goal_state, + actions=actions, + domain=domain) + + +def rush_hour_optimized(): + """ + RUSH-HOUR-PROBLEM (optimized version) + + This version optimizes the planning problem by creating vehicle-specific + actions. Since each vehicle's orientation is fixed, generic predicates like + IsHorizontal can be removed and actions can be created that only apply to the + correct vehicle on its fixed axis of movement. This drastically reduces the + number of permutations the planner needs to generate and check. + """ + # Initial state is simpler as orientation is now baked into the actions. + initial_state = 'At(RedCar, R3, C1) & At(GreenTruck, R1, C4) & At(BlueCar, R5, C2) & ' \ + 'Clear(R1, C1) & Clear(R1, C2) & Clear(R1, C3) & Clear(R1, C5) & Clear(R1, C6) & ' \ + 'Clear(R2, C1) & Clear(R2, C2) & Clear(R2, C3) & Clear(R2, C5) & Clear(R2, C6) & ' \ + 'Clear(R3, C3) & Clear(R3, C4) & Clear(R3, C5) & Clear(R3, C6) & ' \ + 'Clear(R4, C1) & Clear(R4, C2) & Clear(R4, C3) & Clear(R4, C4) & Clear(R4, C5) & Clear(R4, C6) & ' \ + 'Clear(R5, C1) & Clear(R5, C3) & Clear(R5, C4) & Clear(R5, C5) & Clear(R5, C6) & ' \ + 'Clear(R6, C1) & Clear(R6, C3) & Clear(R6, C4) & Clear(R6, C5) & Clear(R6, C6)' + + goal_state = 'At(RedCar, R3, C5)' + + domain = 'Vehicle(RedCar) & Vehicle(GreenTruck) & Vehicle(BlueCar) & ' \ + 'Row(R1) & Row(R2) & Row(R3) & Row(R4) & Row(R5) & Row(R6) & ' \ + 'Col(C1) & Col(C2) & Col(C3) & Col(C4) & Col(C5) & Col(C6) & ' \ + 'NextTo(R1, R2) & NextTo(R2, R3) & NextTo(R3, R4) & NextTo(R4, R5) & NextTo(R5, R6) & ' \ + 'NextTo(C1, C2) & NextTo(C2, C3) & NextTo(C3, C4) & NextTo(C4, C5) & NextTo(C5, C6)' + + actions = [ + # RedCar is horizontal on row R3, length 2 + Action('MoveRedCarRight(c1, c2, c3)', + precond='At(RedCar, R3, c1) & NextTo(c1, c2) & NextTo(c2, c3) & Clear(R3, c3)', + effect='At(RedCar, R3, c2) & ~At(RedCar, R3, c1) & Clear(R3, c1) & ~Clear(R3, c3)', + domain='Col(c1) & Col(c2) & Col(c3)'), + Action('MoveRedCarLeft(c1, c2, c3)', + precond='At(RedCar, R3, c2) & NextTo(c1, c2) & NextTo(c2, c3) & Clear(R3, c1)', + effect='At(RedCar, R3, c1) & ~At(RedCar, R3, c2) & Clear(R3, c3) & ~Clear(R3, c1)', + domain='Col(c1) & Col(c2) & Col(c3)'), + # GreenTruck is vertical on column C4, length 3 + Action('MoveGreenTruckDown(r1, r2, r3, r4)', + precond='At(GreenTruck, r1, C4) & NextTo(r1, r2) & NextTo(r2, r3) & NextTo(r3, r4) & Clear(r4, C4)', + effect='At(GreenTruck, r2, C4) & ~At(GreenTruck, r1, C4) & Clear(r1, C4) & ~Clear(r4, C4)', + domain='Row(r1) & Row(r2) & Row(r3) & Row(r4)'), + Action('MoveGreenTruckUp(r1, r2, r3, r4)', + precond='At(GreenTruck, r2, C4) & NextTo(r1, r2) & NextTo(r2, r3) & NextTo(r3, r4) & Clear(r1, C4)', + effect='At(GreenTruck, r1, C4) & ~At(GreenTruck, r2, C4) & Clear(r4, C4) & ~Clear(r1, C4)', + domain='Row(r1) & Row(r2) & Row(r3) & Row(r4)'), + # BlueCar is vertical on column C2, length 2 + Action('MoveBlueCarDown(r1, r2, r3)', + precond='At(BlueCar, r1, C2) & NextTo(r1, r2) & NextTo(r2, r3) & Clear(r3, C2)', + effect='At(BlueCar, r2, C2) & ~At(BlueCar, r1, C2) & Clear(r1, C2) & ~Clear(r3, C2)', + domain='Row(r1) & Row(r2) & Row(r3)'), + Action('MoveBlueCarUp(r1, r2, r3)', + precond='At(BlueCar, r2, C2) & NextTo(r1, r2) & NextTo(r2, r3) & Clear(r1, C2)', + effect='At(BlueCar, r1, C2) & ~At(BlueCar, r2, C2) & Clear(r3, C2) & ~Clear(r1, C2)', + domain='Row(r1) & Row(r2) & Row(r3)')] + + return PlanningProblem(initial=initial_state, + goals=goal_state, + actions=actions, + domain=domain) + + +def double_tennis_problem(): + """ + [Figure 11.10] DOUBLE-TENNIS-PROBLEM + + A multiagent planning problem involving two partner tennis players + trying to return an approaching ball and repositioning around in the court. + + Example: + >>> from planning import * + >>> dtp = double_tennis_problem() + >>> goal_test(dtp.goals, dtp.initial) + False + >>> dtp.act(expr('Go(A, RightBaseLine, LeftBaseLine)')) + >>> dtp.act(expr('Hit(A, Ball, RightBaseLine)')) + >>> goal_test(dtp.goals, dtp.initial) + False + >>> dtp.act(expr('Go(A, LeftNet, RightBaseLine)')) + >>> goal_test(dtp.goals, dtp.initial) + True + >>> + """ + + return PlanningProblem( + initial='At(A, LeftBaseLine) & At(B, RightNet) & Approaching(Ball, RightBaseLine) & Partner(A, B) & Partner(B, A)', + goals='Returned(Ball) & At(a, LeftNet) & At(a, RightNet)', + actions=[Action('Hit(actor, Ball, loc)', + precond='Approaching(Ball, loc) & At(actor, loc)', + effect='Returned(Ball)'), + Action('Go(actor, to, loc)', + precond='At(actor, loc)', + effect='At(actor, to) & ~At(actor, loc)')]) + + +class ForwardPlan(search.Problem): + """ + [Section 10.2.1] + Forward state-space search + """ + + def __init__(self, planning_problem): + super().__init__(associate('&', planning_problem.initial), associate('&', planning_problem.goals)) + self.planning_problem = planning_problem + self.expanded_actions = self.planning_problem.expand_actions() + + def actions(self, state): + """Return the expanded actions whose preconditions all hold in the given state.""" + return [action for action in self.expanded_actions if all(pre in conjuncts(state) for pre in action.precond)] + + def result(self, state, action): + """Return the state resulting from applying the action to the given state.""" + return associate('&', action(conjuncts(state), action.args).clauses) + + def goal_test(self, state): + """Return True if every goal of the planning problem holds in the given state.""" + return all(goal in conjuncts(state) for goal in self.planning_problem.goals) + + def h(self, state): + """ + Computes ignore delete lists heuristic by creating a relaxed version of the original problem (we can do that + by removing the delete lists from all actions, i.e. removing all negative literals from effects) that will be + easier to solve through GraphPlan and where the length of the solution will serve as a good heuristic. + """ + relaxed_planning_problem = PlanningProblem(initial=state.state, + goals=self.goal, + actions=[action.relaxed() for action in + self.planning_problem.actions]) + try: + return len(linearize(GraphPlan(relaxed_planning_problem).execute())) + except: + return np.inf + + +class BackwardPlan(search.Problem): + """ + [Section 10.2.2] + Backward relevant-states search + """ + + def __init__(self, planning_problem): + super().__init__(associate('&', planning_problem.goals), associate('&', planning_problem.initial)) + self.planning_problem = planning_problem + self.expanded_actions = self.planning_problem.expand_actions() + + def actions(self, subgoal): + """ + Returns True if the action is relevant to the subgoal, i.e.: + - the action achieves an element of the effects + - the action doesn't delete something that needs to be achieved + - the preconditions are consistent with other subgoals that need to be achieved + """ + + def negate_clause(clause): + return Expr(clause.op.replace('Not', ''), *clause.args) if clause.op[:3] == 'Not' else Expr( + 'Not' + clause.op, *clause.args) + + subgoal = conjuncts(subgoal) + return [action for action in self.expanded_actions if + (any(prop in action.effect for prop in subgoal) and + not any(negate_clause(prop) in subgoal for prop in action.effect) and + not any(negate_clause(prop) in subgoal and negate_clause(prop) not in action.effect + for prop in action.precond))] + + def result(self, subgoal, action): + """Regress the subgoal through the action, i.e. ``g' = (g - effects(a)) + preconds(a)``.""" + # g' = (g - effects(a)) + preconds(a) + return associate('&', set(set(conjuncts(subgoal)).difference(action.effect)).union(action.precond)) + + def goal_test(self, subgoal): + """Return True if the subgoal is entailed by the search goal (the problem's initial state).""" + return all(goal in conjuncts(self.goal) for goal in conjuncts(subgoal)) + + def h(self, subgoal): + """ + Computes ignore delete lists heuristic by creating a relaxed version of the original problem (we can do that + by removing the delete lists from all actions, i.e. removing all negative literals from effects) that will be + easier to solve through GraphPlan and where the length of the solution will serve as a good heuristic. + """ + relaxed_planning_problem = PlanningProblem(initial=self.goal, + goals=subgoal.state, + actions=[action.relaxed() for action in + self.planning_problem.actions]) + try: + return len(linearize(GraphPlan(relaxed_planning_problem).execute())) + except: + return np.inf + + +def CSPlan(planning_problem, solution_length, CSP_solver=ac_search_solver, arc_heuristic=sat_up): + """ + [Section 10.4.3] + Planning as Constraint Satisfaction Problem + """ + + def st(var, stage): + """Returns a string for the var-stage pair that can be used as a variable""" + return str(var) + "_" + str(stage) + + def if_(v1, v2): + """If the second argument is v2, the first argument must be v1""" + + def if_fun(x1, x2): + return x1 == v1 if x2 == v2 else True + + if_fun.__name__ = "if the second argument is " + str(v2) + " then the first argument is " + str(v1) + " " + return if_fun + + def eq_if_not_in_(actset): + """First and third arguments are equal if action is not in actset""" + + def eq_if_not_in(x1, a, x2): + return x1 == x2 if a not in actset else True + + eq_if_not_in.__name__ = "first and third arguments are equal if action is not in " + str(actset) + " " + return eq_if_not_in + + expanded_actions = planning_problem.expand_actions() + fluent_values = planning_problem.expand_fluents() + for horizon in range(solution_length): + act_vars = [st('action', stage) for stage in range(horizon + 1)] + domains = {av: list(map(lambda action: expr(str(action)), expanded_actions)) for av in act_vars} + domains.update({st(var, stage): {True, False} for var in fluent_values for stage in range(horizon + 2)}) + # initial state constraints + constraints = [Constraint((st(var, 0),), is_constraint(val)) + for (var, val) in {expr(str(fluent).replace('Not', '')): + True if fluent.op[:3] != 'Not' else False + for fluent in planning_problem.initial}.items()] + constraints += [Constraint((st(var, 0),), is_constraint(False)) + for var in {expr(str(fluent).replace('Not', '')) + for fluent in fluent_values if fluent not in planning_problem.initial}] + # goal state constraints + constraints += [Constraint((st(var, horizon + 1),), is_constraint(val)) + for (var, val) in {expr(str(fluent).replace('Not', '')): + True if fluent.op[:3] != 'Not' else False + for fluent in planning_problem.goals}.items()] + # precondition constraints + constraints += [Constraint((st(var, stage), st('action', stage)), if_(val, act)) + # st(var, stage) == val if st('action', stage) == act + for act, strps in {expr(str(action)): action for action in expanded_actions}.items() + for var, val in {expr(str(fluent).replace('Not', '')): + True if fluent.op[:3] != 'Not' else False + for fluent in strps.precond}.items() + for stage in range(horizon + 1)] + # effect constraints + constraints += [Constraint((st(var, stage + 1), st('action', stage)), if_(val, act)) + # st(var, stage + 1) == val if st('action', stage) == act + for act, strps in {expr(str(action)): action for action in expanded_actions}.items() + for var, val in {expr(str(fluent).replace('Not', '')): True if fluent.op[:3] != 'Not' else False + for fluent in strps.effect}.items() + for stage in range(horizon + 1)] + # frame constraints + constraints += [Constraint((st(var, stage), st('action', stage), st(var, stage + 1)), + eq_if_not_in_(set(map(lambda action: expr(str(action)), + {act for act in expanded_actions if var in act.effect + or Expr('Not' + var.op, *var.args) in act.effect})))) + for var in fluent_values for stage in range(horizon + 1)] + csp = NaryCSP(domains, constraints) + sol = CSP_solver(csp, arc_heuristic=arc_heuristic) + if sol: + return [sol[a] for a in act_vars] + + +def SATPlan(planning_problem, solution_length, SAT_solver=cdcl_satisfiable): + """ + [Section 10.4.1] + Planning as Boolean satisfiability + """ + + def expand_transitions(state, actions): + state = sorted(conjuncts(state)) + for action in filter(lambda act: act.check_precond(state, act.args), actions): + transition[associate('&', state)].update( + {Expr(action.name, *action.args): + associate('&', sorted(set(filter(lambda clause: clause.op[:3] != 'Not', + action(state, action.args).clauses)))) + if planning_problem.is_strips() + else associate('&', sorted(set(action(state, action.args).clauses)))}) + for state in transition[associate('&', state)].values(): + if state not in transition: + expand_transitions(expr(state), actions) + + transition = defaultdict(dict) + expand_transitions(associate('&', planning_problem.initial), planning_problem.expand_actions()) + + return SAT_plan(associate('&', sorted(planning_problem.initial)), transition, + associate('&', sorted(planning_problem.goals)), solution_length, SAT_solver=SAT_solver) + + +def predicate_negate(e): + """Return the logical negation of an Expr predicate, avoiding a double 'Not' prefix.""" + return Expr(e.op[3:], *e.args) if e.op.startswith('Not') else Expr('Not' + e.op, *e.args) + + +class Level: + """ + Contains the state of the planning problem + and exhaustive list of actions which use the + states as pre-condition. + """ + + def __init__(self, kb): + """Initializes variables to hold state and action details of a level""" + + self.kb = kb + # current state + self.current_state = kb.clauses + # current action to state link + # Action -> preconditions for that action + self.current_action_links = {} + # current state to action link + # Precondition -> what is applicable (actions) + self.current_state_links = {} + # current action to next state link (e.g. Go(Home, HW) --> At(HW) and NotAt(Home)) + # aka forward link in time (dependency) + self.next_action_links = {} + # next state to current action link (e.g. NotAt(Home): [Go(Home, HW), Go(Home, SM)]) + # aka backwards link in time (dependency) + self.next_state_links = {} + # mutually exclusive actions + self.action_mutexes = [] + # mutually exclusive states + self.state_mutexes = [] + + def __call__(self, actions, objects): + self.build(actions, objects) + self.find_mutex() + + def __str__(self): + state_str = ', '.join(str(s) for s in self.current_state) + action_str = ', '.join(str(a) for a in self.current_action_links.keys()) + mutex_str = ', '.join(str(m) for m in self.action_mutexes) + return ('\n' + ' Current State: {{{}}}\n' + ' Actions: {{{}}}\n' + ' Mutex: {{{}}}\n'.format(state_str, action_str, mutex_str)) + + __repr__ = __str__ + + def separate(self, e): + """Separates an iterable of elements into positive and negative parts""" + + positive = [] + negative = [] + for clause in e: + if clause.op[:3] == 'Not': + negative.append(clause) + else: + positive.append(clause) + return positive, negative + + def find_mutex(self): + """Finds mutually exclusive actions""" + + # clear out effects from state mutex prior computation + self.action_mutexes = [] + + # Competing needs - two actions are mutex if any of their preconditions + # are mutex at the previous state level + for a1, a2 in itertools.combinations(self.current_action_links.keys(), 2): + preconds_a1 = self.current_action_links[a1] + preconds_a2 = self.current_action_links[a2] + + if any({p, q} in self.state_mutexes for p in preconds_a1 for q in preconds_a2): + mutex_pair = {a1, a2} + if mutex_pair not in self.action_mutexes: + self.action_mutexes.append(mutex_pair) + + # Interference and inconsistent effects mutex calculation + for a1, a2 in itertools.combinations(self.next_action_links.keys(), 2): + preconds_a1 = self.current_action_links.get(a1, []) + preconds_a2 = self.current_action_links.get(a2, []) + effects_a1 = self.next_action_links.get(a1, []) + effects_a2 = self.next_action_links.get(a2, []) + + interference = False + # Interference check + for p1 in preconds_a1: + if predicate_negate(p1) in effects_a2: + interference = True + for p2 in preconds_a2: + if predicate_negate(p2) in effects_a1: + interference = True + + # Inconsistent effects check + for e1 in effects_a1: + if predicate_negate(e1) in effects_a2: + interference = True + for e2 in effects_a2: + if predicate_negate(e2) in effects_a1: + interference = True + + if interference: + mutex_pair = {a1, a2} + if mutex_pair not in self.action_mutexes: + self.action_mutexes.append(mutex_pair) + + def populate_prop_mutexes(self): + """Compute the next level's proposition mutexes based on the current action mutexes""" + + # Inconsistent support - two props cannot be true given competing supporting actions + state_mutex = [] + next_state_pairs = itertools.combinations(self.next_state_links.keys(), 2) + for next_state_pair in list(next_state_pairs): + s1, s2 = list(next_state_pair) + acts_to_s1 = self.next_state_links.get(s1, []) + acts_to_s2 = self.next_state_links.get(s2, []) + + # ensure our mutexes only apply to pairs, not single states + if acts_to_s1 == [] or acts_to_s2 == []: + continue + + # if any two actions that lead to these states are not mutex, + # do not add a mutex to these states + if all({a1, a2} in self.action_mutexes or {a2, a1} in self.action_mutexes + for a1 in acts_to_s1 for a2 in acts_to_s2): + mutex_pair = {s1, s2} + if mutex_pair not in state_mutex: + state_mutex.append(mutex_pair) + + # If there are pairs of propositions that are negations of each other, they must be mutex + for s1i in range(len(self.current_state)): + for s2i in range(s1i, len(self.current_state)): + s1, s2 = self.current_state[s1i], self.current_state[s2i] + if (repr(s2)[0:3] == 'Not' and repr(s1) == repr(s2)[3:] or + repr(s1)[0:3] == 'Not' and repr(s1)[3:] == repr(s2)): + mutex_pair = {s1, s2} + if mutex_pair not in state_mutex: + state_mutex.append(mutex_pair) + + return state_mutex + + def prune_invalid_actions(self): + """Remove actions whose own preconditions are mutex (unsupportable)""" + + to_remove = [] + + # Normalize state mutex set for fast membership checks + state_mutex_lookup = set() + for m in self.state_mutexes: + state_mutex_lookup.add(frozenset(m)) + + for action, preconds in list(self.current_action_links.items()): + invalid = False + for p1, p2 in itertools.combinations(preconds, 2): + if frozenset({p1, p2}) in state_mutex_lookup: + invalid = True + break + if invalid: + to_remove.append(action) + + # Remove invalid actions from all mappings + for action in to_remove: + # forward mappings + self.current_action_links.pop(action, None) + self.next_action_links.pop(action, None) + + # reverse mapping: state -> actions (current_state_links) + for precond in list(self.current_state_links.keys()): + actions_for_pre = self.current_state_links.get(precond, []) + if action in actions_for_pre: + actions_for_pre.remove(action) + if not actions_for_pre: + self.current_state_links.pop(precond, None) + else: + self.current_state_links[precond] = actions_for_pre + + # reverse mapping: next_state -> actions (next_state_links) + for effect in list(self.next_state_links.keys()): + actions_for_eff = self.next_state_links.get(effect, []) + if action in actions_for_eff: + actions_for_eff.remove(action) + if not actions_for_eff: + self.next_state_links.pop(effect, None) + else: + self.next_state_links[effect] = actions_for_eff + + def build(self, actions, objects): + """Populates the lists and dictionaries containing the state action dependencies""" + + for clause in self.current_state: + p_expr = Expr('P' + clause.op, *clause.args) + self.current_action_links[p_expr] = [clause] + self.next_action_links[p_expr] = [clause] + self.current_state_links[clause] = [p_expr] + self.next_state_links[clause] = [p_expr] + + for a in actions: + num_args = len(a.args) + possible_args = tuple(itertools.permutations(objects, num_args)) + + for arg in possible_args: + if a.check_precond(self.kb, arg): + for num, symbol in enumerate(a.args): + if not symbol.op.islower(): + arg = list(arg) + arg[num] = symbol + arg = tuple(arg) + + new_action = a.substitute(Expr(a.name, *a.args), arg) + self.current_action_links[new_action] = [] + + for clause in a.precond: + new_clause = a.substitute(clause, arg) + self.current_action_links[new_action].append(new_clause) + if new_clause in self.current_state_links: + self.current_state_links[new_clause].append(new_action) + else: + self.current_state_links[new_clause] = [new_action] + + self.next_action_links[new_action] = [] + for clause in a.effect: + new_clause = a.substitute(clause, arg) + + self.next_action_links[new_action].append(new_clause) + if new_clause in self.next_state_links: + self.next_state_links[new_clause].append(new_action) + else: + self.next_state_links[new_clause] = [new_action] + + def perform_actions(self): + """Performs the necessary actions and returns a new Level""" + + new_kb = FolKB(list(set(self.next_state_links.keys()))) + return Level(new_kb) + + +class Graph: + """ + Contains levels of state and actions + Used in graph planning algorithm to extract a solution + """ + + def __init__(self, planning_problem): + self.planning_problem = planning_problem + self.kb = FolKB(planning_problem.initial) + self.levels = [Level(self.kb)] + self.objects = set(arg for clause in self.kb.clauses for arg in clause.args) + + def __call__(self): + self.expand_graph() + + def __str__(self): + levels_str = '\n'.join('Level {}:\n{}'.format(i, level) + for i, level in enumerate(self.levels)) + return '\n Objects: {}\n{}\n'.format(self.objects, levels_str) + + __repr__ = __str__ + + def expand_graph(self): + """Expands the graph by a level""" + + last_level = self.levels[-1] + # populate state/actions/mutexes + last_level(self.planning_problem.actions, self.objects) + last_level.prune_invalid_actions() + # create new level + new_level = last_level.perform_actions() + # populate the mutexes for the next state level to come + new_level.state_mutexes = last_level.populate_prop_mutexes() + self.levels.append(new_level) + + def non_mutex_goals(self, goals, index): + """Checks whether the goals are mutually exclusive""" + + goal_perm = itertools.combinations(goals, 2) + for g in goal_perm: + if set(g) in self.levels[index].state_mutexes: + return False + return True + + +class GraphPlan: + """ + Class for formulation GraphPlan algorithm + Constructs a graph of state and action space + Returns solution for the planning problem + """ + + def __init__(self, planning_problem): + self.graph = Graph(planning_problem) + self.no_goods = [] + self.solution = [] + + def __str__(self): + sol_str = ('No solution found' if not self.solution + else 'Solution with {} steps'.format(len(self.solution))) + return '\n Nogoods: {}\n {}\n'.format(len(self.no_goods), sol_str) + + __repr__ = __str__ + + def check_leveloff(self): + """Checks if the graph has leveled off""" + + if len(self.graph.levels) < 2: + return False + + level = self.graph.levels[-1] + prev_level = self.graph.levels[-2] + + same_state = set(level.current_state) == set(prev_level.current_state) + + level_mutex = set(frozenset(m) for m in level.state_mutexes) + prev_mutex = set(frozenset(m) for m in prev_level.state_mutexes) + same_mutex = level_mutex == prev_mutex + + return same_state and same_mutex + + def get_preconditions_for(self, action_set, level): + """Collects all unique preconditions for a given set of actions in a level""" + + all_preconditions = set() + for action in action_set: + preconditions = level.current_action_links.get(action, []) + all_preconditions.update(preconditions) + return all_preconditions + + def find_valid_action_sets(self, goals, level): + """ + Finds sets of actions in the given level that are not mutually exclusive + and that collectively satisfy all the goals. + """ + + valid_sets = [] + + actions_for_goal = {g: level.next_state_links.get(g, []) for g in goals} + potential_action_groups = [actions_for_goal[g] for g in goals] + + for action_combination in itertools.product(*potential_action_groups): + action_set = set(action_combination) + + is_mutex = False + for a1, a2 in itertools.combinations(action_set, 2): + if {a1, a2} in level.action_mutexes: + is_mutex = True + break + + if not is_mutex and action_set not in valid_sets: + valid_sets.append(action_set) + + return valid_sets + + def extract_solution(self, goals): + """ + Starts the solution extraction process by calling the recursive helper + and returning the final plan. + """ + + return self.extract_solution_recursive(set(goals), len(self.graph.levels) - 1) + + def extract_solution_recursive(self, goals, level_index): + """ + Recursively searches for a plan backwards from a given proposition level. + + goals is the set of goal propositions to satisfy and level_index is the + index of the proposition level currently being solved. + """ + + # Base case: we have recursed back to the initial proposition layer (level 0). + # The goals at this point are the preconditions for the very first set of + # actions, so we just check whether they hold in the initial state. + if level_index == 0: + initial_state = set(self.graph.levels[0].current_state) + if goals.issubset(initial_state): + return [] # success, return the empty plan to be built upon + else: + return None # failure, preconditions are not met + + # Memoization: check whether we already proved this subproblem unsolvable. + if (level_index, frozenset(goals)) in self.no_goods: + return None + + # Recursive step: to satisfy the goals at level_index we need a set of + # non-mutex actions from the previous level's action layer. + action_level = self.graph.levels[level_index - 1] + valid_action_sets = self.find_valid_action_sets(goals, action_level) + + for action_set in valid_action_sets: + # The new sub-goals are the combined preconditions for this action set. + new_goals = self.get_preconditions_for(action_set, action_level) + + # Recurse to solve for the new goals at the previous proposition layer. + sub_plan = self.extract_solution_recursive(new_goals, level_index - 1) + + if sub_plan is not None: + return sub_plan + [list(action_set)] + + # No solution from this subproblem, so record it as a no-good. + nogood_item = (level_index, frozenset(goals)) + if nogood_item not in self.no_goods: + self.no_goods.append(nogood_item) + return None + + def goal_test(self, kb): + """Return True if all of the problem's goals can be proven from the knowledge base.""" + return all(kb.ask(q) is not False for q in self.graph.planning_problem.goals) + + def execute(self): + """Executes the GraphPlan algorithm for the given problem""" + + while True: + self.graph.expand_graph() + if (self.goal_test(self.graph.levels[-1].kb) and + self.graph.non_mutex_goals(self.graph.planning_problem.goals, -1)): + solution = self.extract_solution(self.graph.planning_problem.goals) + if solution: + return [solution] + + if self.check_leveloff(): + return None + + +class Linearize: + """ + Coordinator that linearizes partially ordered solutions generated by a GraphPlan object. + """ + + def __init__(self, planning_problem): + self.planning_problem = planning_problem + + def filter(self, solution): + """Filter out persistence actions from a solution""" + + new_solution = [] + for section in solution: + new_section = [] + for operation in section: + if not (operation.op[0] == 'P' and operation.op[1].isupper()): + new_section.append(operation) + # filter may remove all actions if all actions are persistent + if new_section != []: + new_solution.append(new_section) + return new_solution + + def orderlevel(self, level, planning_problem): + """Return valid linear order of actions for a given level""" + + for permutation in itertools.permutations(level): + temp = copy.deepcopy(planning_problem) + count = 0 + for action in permutation: + try: + temp.act(action) + count += 1 + except: + count = 0 + temp = copy.deepcopy(planning_problem) + continue + if count == len(permutation): + return list(permutation), temp + # identifying a linear ordering for the level failed + return None, planning_problem + + def execute(self): + """Finds a total-order solution for a planning graph (not necessarily unique)""" + + graph_plan_solution = GraphPlan(self.planning_problem).execute() + + # exit if no plan found + if graph_plan_solution is None: + return None + + ordered_solution = None + for possible_plan in graph_plan_solution: + filtered_solution = self.filter(possible_plan) + + ordered_solution = [] + # planning_problem maintains the current state as we iterate over the + # levels, allowing test application of the actions + planning_problem = self.planning_problem + for level in filtered_solution: + level_solution, planning_problem = self.orderlevel(level, planning_problem) + if not level_solution: + # level failed to apply, this plan does not work + ordered_solution = None + break + + for element in level_solution: + ordered_solution.append(element) + + if not ordered_solution: + continue + else: + break + + return ordered_solution + + +def linearize(solution): + """Converts a level-ordered solution into a linear solution""" + + linear_solution = [] + for section in solution[0]: + for operation in section: + if not (operation.op[0] == 'P' and operation.op[1].isupper()): + linear_solution.append(operation) + + return linear_solution + + +class PartialOrderPlanner: + """ + [Section 10.13] PARTIAL-ORDER-PLANNER + + Partially ordered plans are created by a search through the space of plans + rather than a search through the state space. It views planning as a refinement of partially ordered plans. + A partially ordered plan is defined by a set of actions and a set of constraints of the form A < B, + which denotes that action A has to be performed before action B. + To summarize the working of a partial order planner, + + 1. An open precondition is selected (a sub-goal that we want to achieve). + 2. An action that fulfils the open precondition is chosen. + 3. Temporal constraints are updated. + 4. Existing causal links are protected. Protection is a method that checks if the causal links conflict + and if they do, temporal constraints are added to fix the threats. + 5. The set of open preconditions is updated. + 6. Temporal constraints of the selected action and the next action are established. + 7. A new causal link is added between the selected action and the owner of the open precondition. + 8. The set of new causal links is checked for threats and if found, the threat is removed by either promotion or + demotion. If promotion or demotion is unable to solve the problem, the planning problem cannot be solved with + the current sequence of actions or it may not be solvable at all. + 9. These steps are repeated until the set of open preconditions is empty. + """ + + def __init__(self, planning_problem): + self.tries = 1 + # safety bounds for the backtracking search in execute(): the maximum + # number of actions a plan may contain (iterative-deepening target) and + # the maximum number of node expansions per deepening level + self._max_plan_actions = 12 + self._max_expansions = 20000 + self.planning_problem = planning_problem + self.causal_links = [] + self.start = Action('Start', [], self.planning_problem.initial) + self.finish = Action('Finish', self.planning_problem.goals, []) + self.actions = set() + self.actions.add(self.start) + self.actions.add(self.finish) + self.constraints = set() + self.constraints.add((self.start, self.finish)) + self.agenda = set() + for precond in self.finish.precond: + self.agenda.add((precond, self.finish)) + self.expanded_actions = planning_problem.expand_actions() + + def find_open_precondition(self): + """ + Find the open precondition with the least number of achieving actions + (a most-constrained-variable heuristic). Returns the triple + (precondition, action_that_needs_it, [achieving_actions]). Iteration is + ordered deterministically so the search does not depend on set/hash + ordering. Returns (None, None, None) when some open precondition has no + achiever at all, which is a dead end for the current partial plan. + """ + possible_actions = list(self.actions) + self.expanded_actions + number_of_ways = dict() + actions_for_precondition = dict() + for open_precondition, act in sorted(self.agenda, key=str): + if open_precondition in number_of_ways: + continue + achievers = [action for action in possible_actions + if any(effect == open_precondition for effect in action.effect)] + if not achievers: + return None, None, None + number_of_ways[open_precondition] = len(achievers) + actions_for_precondition[open_precondition] = achievers + + if not number_of_ways: + return None, None, None + + chosen = min(number_of_ways, key=lambda p: (number_of_ways[p], str(p))) + act1 = next(act for precond, act in sorted(self.agenda, key=str) if precond == chosen) + return chosen, act1, actions_for_precondition[chosen] + + def find_action_for_precondition(self, oprec): + """Find action for a given precondition""" + + # either + # choose act0 E Actions such that act0 achieves G + for action in self.actions: + for effect in action.effect: + if effect == oprec: + return action, 0 + + # or + # choose act0 E Actions such that act0 achieves G + for action in self.planning_problem.actions: + for effect in action.effect: + if effect.op == oprec.op: + bindings = unify_mm(effect, oprec) + if bindings is None: + break + return action, bindings + + def generate_expr(self, clause, bindings): + """Generate atomic expression from generic expression given variable bindings""" + + new_args = [] + for arg in clause.args: + if arg in bindings: + new_args.append(bindings[arg]) + else: + new_args.append(arg) + + try: + return Expr(str(clause.name), *new_args) + except: + return Expr(str(clause.op), *new_args) + + def generate_action_object(self, action, bindings): + """Generate action object given a generic action and variable bindings""" + + # if bindings is 0, it means the action already exists in self.actions + if bindings == 0: + return action + + # bindings cannot be None + else: + new_expr = self.generate_expr(action, bindings) + new_preconds = [] + for precond in action.precond: + new_precond = self.generate_expr(precond, bindings) + new_preconds.append(new_precond) + new_effects = [] + for effect in action.effect: + new_effect = self.generate_expr(effect, bindings) + new_effects.append(new_effect) + return Action(new_expr, new_preconds, new_effects) + + def cyclic(self, graph): + """Check cyclicity of a directed graph""" + + new_graph = dict() + for element in graph: + if element[0] in new_graph: + new_graph[element[0]].append(element[1]) + else: + new_graph[element[0]] = [element[1]] + + path = set() + + def visit(vertex): + path.add(vertex) + for neighbor in new_graph.get(vertex, ()): + if neighbor in path or visit(neighbor): + return True + path.remove(vertex) + return False + + value = any(visit(v) for v in new_graph) + return value + + def add_const(self, constraint, constraints): + """Add the constraint to constraints if the resulting graph is acyclic""" + + if constraint[0] == self.finish or constraint[1] == self.start: + return constraints + + new_constraints = set(constraints) + new_constraints.add(constraint) + + if self.cyclic(new_constraints): + return constraints + return new_constraints + + def is_a_threat(self, precondition, effect): + """Check if effect is a threat to precondition""" + + if (str(effect.op) == 'Not' + str(precondition.op)) or ('Not' + str(effect.op) == str(precondition.op)): + if effect.args == precondition.args: + return True + return False + + def protect(self, causal_link, action, constraints): + """Check and resolve threats by promotion or demotion""" + + threat = False + for effect in action.effect: + if self.is_a_threat(causal_link[1], effect): + threat = True + break + + if action != causal_link[0] and action != causal_link[2] and threat: + # try promotion + new_constraints = set(constraints) + new_constraints.add((action, causal_link[0])) + if not self.cyclic(new_constraints): + constraints = self.add_const((action, causal_link[0]), constraints) + else: + # try demotion + new_constraints = set(constraints) + new_constraints.add((causal_link[2], action)) + if not self.cyclic(new_constraints): + constraints = self.add_const((causal_link[2], action), constraints) + else: + # both promotion and demotion fail + print('Unable to resolve a threat caused by', action, 'onto', causal_link) + return + return constraints + + def convert(self, constraints): + """Convert constraints into a dict of Action to set orderings""" + + graph = dict() + for constraint in constraints: + if constraint[0] in graph: + graph[constraint[0]].add(constraint[1]) + else: + graph[constraint[0]] = set() + graph[constraint[0]].add(constraint[1]) + return graph + + def toposort(self, graph): + """Generate topological ordering of constraints""" + + if len(graph) == 0: + return + + graph = graph.copy() + + for k, v in graph.items(): + v.discard(k) + + extra_elements_in_dependencies = _reduce(set.union, graph.values()) - set(graph.keys()) + + graph.update({element: set() for element in extra_elements_in_dependencies}) + while True: + ordered = set(element for element, dependency in graph.items() if len(dependency) == 0) + if not ordered: + break + yield ordered + graph = {element: (dependency - ordered) + for element, dependency in graph.items() + if element not in ordered} + if len(graph) != 0: + raise ValueError('The graph is not acyclic and cannot be linearly ordered') + + def display_plan(self): + """Display causal links, constraints and the plan""" + + print('Causal Links') + for causal_link in self.causal_links: + print(causal_link) + + print('\n_constraints') + for constraint in self.constraints: + print(constraint[0], '<', constraint[1]) + + print('\n_partial Order Plan') + print(list(reversed(list(self.toposort(self.convert(self.constraints)))))) + + def execute(self, display=True): + """ + Execute the algorithm with backtracking, using iterative deepening on the + number of actions in the plan. The original greedy version committed to + the first achiever it happened to iterate over and could not recover when + that action's own preconditions turned out to be unsatisfiable, so it + depended on hash ordering and often printed 'Probably Wrong' / "Couldn't + find a solution". Backtracking over both action choices and threat + resolution (promotion vs demotion), together with the deterministic + selection in find_open_precondition and a smallest-plan-first deepening + bound, makes the planner solve the standard problems reproducibly and + return a short, valid plan. + """ + pristine = self._snapshot() + for limit in range(1, self._max_plan_actions + 1): + self._restore(pristine) + if self._search([self._max_expansions], limit): + if display: + self.display_plan() + else: + return self.constraints, self.causal_links + return + print("Couldn't find a solution") + if not display: + return None, None + + def _reachable(self, source, target): + """True if target is forced to come after source by the ordering constraints""" + + stack, seen = [source], set() + while stack: + node = stack.pop() + if node == target: + return True + if node in seen: + continue + seen.add(node) + stack.extend(b for a, b in self.constraints if a == node) + return False + + def _open_threat(self): + """ + Return an (action, causal_link) threat that is not yet resolved by the + ordering constraints, or None if every causal link is protected. A + causal link (a0, p, a1) is threatened by an action whose effect negates p + unless the action is already ordered before a0 (promotion) or after a1 + (demotion). + """ + for a0, p, a1 in self.causal_links: + for action in self.actions: + if action == a0 or action == a1: + continue + if any(self.is_a_threat(p, effect) for effect in action.effect): + if not (self._reachable(action, a0) or self._reachable(a1, action)): + return action, (a0, p, a1) + return None + + def _snapshot(self): + return set(self.actions), set(self.constraints), list(self.causal_links), set(self.agenda) + + def _restore(self, snapshot): + self.actions, self.constraints, self.causal_links, self.agenda = ( + set(snapshot[0]), set(snapshot[1]), list(snapshot[2]), set(snapshot[3])) + + def _search(self, budget, limit): + """ + Recursively complete the partial plan, backtracking on failure. Three + kinds of choice points are explored: which action satisfies an open + precondition, how each threat is resolved (promotion vs demotion), and - + bounded by 'limit' - whether to introduce a new action at all. Returns + True and leaves the solution in self.* on success. + """ + if budget[0] <= 0: + return False + budget[0] -= 1 + + # first, resolve any outstanding threat to a causal link (choice point) + threat = self._open_threat() + if threat is not None: + action, (a0, p, a1) = threat + snapshot = self._snapshot() + for ordering in ((action, a0), (a1, action)): # promotion, then demotion + new_constraints = self.add_const(ordering, self.constraints) + if ordering in new_constraints: # ordering was consistent (acyclic and allowed) + self.constraints = new_constraints + if self._search(budget, limit): + return True + self._restore(snapshot) + return False + + # no open threats: a plan with an empty agenda is a complete solution + if not self.agenda: + return True + + # select from the agenda (most-constrained precondition first) + G, act1, possible_actions = self.find_open_precondition() + if G is None: # an open precondition has no achiever -> dead end + return False + + # number of actions already introduced, excluding the dummy Start/Finish + introduced = len(self.actions) - 2 + snapshot = self._snapshot() + # try each achiever deterministically, reusing existing actions first + for act0 in sorted(set(possible_actions), key=lambda a: (a not in self.actions, str(a))): + is_new = act0 not in self.actions + if is_new and introduced >= limit: # deepening bound on plan size + continue + self.agenda.discard((G, act1)) + self.actions.add(act0) + self.constraints = self.add_const((self.start, act0), self.constraints) + self.constraints = self.add_const((act0, act1), self.constraints) + # the causal link act0 --G--> act1 requires act0 strictly before act1 + # (and after start); add_const drops an ordering that would create a + # cycle, so reject the choice when the required ordering is not enforced + if ((act0 == act1 or self._reachable(act0, act1)) and + (act0 == self.start or self._reachable(self.start, act0))): + if (act0, G, act1) not in self.causal_links: + self.causal_links.append((act0, G, act1)) + if is_new: # a freshly introduced action contributes its own preconditions + for precondition in act0.precond: + self.agenda.add((precondition, act0)) + if self._search(budget, limit): + return True + # undo and try the next achiever + self._restore(snapshot) + return False + + +def spare_tire_graph_plan(): + """Solves the spare tire problem using GraphPlan""" + return GraphPlan(spare_tire()).execute() + + +def three_block_tower_graph_plan(): + """Solves the Sussman Anomaly problem using GraphPlan""" + return GraphPlan(three_block_tower()).execute() + + +def air_cargo_graph_plan(): + """Solves the air cargo problem using GraphPlan""" + return GraphPlan(air_cargo()).execute() + + +def have_cake_and_eat_cake_too_graph_plan(): + """Solves the cake problem using GraphPlan""" + return GraphPlan(have_cake_and_eat_cake_too()).execute() + + +def shopping_graph_plan(): + """Solves the shopping problem using GraphPlan""" + return GraphPlan(shopping_problem()).execute() + + +def socks_and_shoes_graph_plan(): + """Solves the socks and shoes problem using GraphPlan""" + return GraphPlan(socks_and_shoes()).execute() + + +def simple_blocks_world_graph_plan(): + """Solves the simple blocks world problem""" + return GraphPlan(simple_blocks_world()).execute() + + +class HLA(Action): + """ + Define Actions for the real-world (that may be refined further), and satisfy resource + constraints. + """ + unique_group = 1 + + def __init__(self, action, precond=None, effect=None, duration=0, consume=None, use=None): + """ + As opposed to actions, to define HLA, we have added constraints. + duration holds the amount of time required to execute the task + consumes holds a dictionary representing the resources the task consumes + uses holds a dictionary representing the resources the task uses + """ + precond = precond or [None] + effect = effect or [None] + super().__init__(action, precond, effect) + self.duration = duration + self.consumes = consume or {} + self.uses = use or {} + self.completed = False + # self.priority = -1 # must be assigned in relation to other HLAs + # self.job_group = -1 # must be assigned in relation to other HLAs + + def do_action(self, job_order, available_resources, kb, args): + """ + An HLA based version of act - along with knowledge base updation, it handles + resource checks, and ensures the actions are executed in the correct order. + """ + if not self.has_usable_resource(available_resources): + raise Exception('Not enough usable resources to execute {}'.format(self.name)) + if not self.has_consumable_resource(available_resources): + raise Exception('Not enough consumable resources to execute {}'.format(self.name)) + if not self.inorder(job_order): + raise Exception("Can't execute {} - execute prerequisite actions first". + format(self.name)) + kb = super().act(kb, args) # update knowledge base + for resource in self.consumes: # remove consumed resources + available_resources[resource] -= self.consumes[resource] + self.completed = True # set the task status to complete + return kb + + def has_consumable_resource(self, available_resources): + """ + Ensure there are enough consumable resources for this action to execute. + """ + for resource in self.consumes: + if available_resources.get(resource) is None: + return False + if available_resources[resource] < self.consumes[resource]: + return False + return True + + def has_usable_resource(self, available_resources): + """ + Ensure there are enough usable resources for this action to execute. + """ + for resource in self.uses: + if available_resources.get(resource) is None: + return False + if available_resources[resource] < self.uses[resource]: + return False + return True + + def inorder(self, job_order): + """ + Ensure that all the jobs that had to be executed before the current one have been + successfully executed. + """ + for jobs in job_order: + if self in jobs: + for job in jobs: + if job is self: + return True + if not job.completed: + return False + return True + + +class RealWorldPlanningProblem(PlanningProblem): + """ + Define real-world problems by aggregating resources as numerical quantities instead of + named entities. + + This class is identical to PDDL, except that it overloads the act function to handle + resource and ordering conditions imposed by HLA as opposed to Action. + """ + + def __init__(self, initial, goals, actions, jobs=None, resources=None): + super().__init__(initial, goals, actions) + self.jobs = jobs + self.resources = resources or {} + + def act(self, action): + """ + Performs the HLA given as argument. + + Note that this is different from the superclass action - where the parameter was an + Expression. For real world problems, an Expr object isn't enough to capture all the + detail required for executing the action - resources, preconditions, etc need to be + checked for too. + """ + args = action.args + list_action = first(a for a in self.actions if a.name == action.name) + if list_action is None: + raise Exception("Action '{}' not found".format(action.name)) + self.initial = list_action.do_action(self.jobs, self.resources, self.initial, args).clauses + + def refinements(self, library): # refinements may be (multiple) HLA themselves ... + """ + State is a Problem, containing the current state kb library is a + dictionary containing details for every possible refinement. e.g.:: + + { + 'HLA': [ + 'Go(Home, SFO)', + 'Go(Home, SFO)', + 'Drive(Home, SFOLongTermParking)', + 'Shuttle(SFOLongTermParking, SFO)', + 'Taxi(Home, SFO)' + ], + 'steps': [ + ['Drive(Home, SFOLongTermParking)', 'Shuttle(SFOLongTermParking, SFO)'], + ['Taxi(Home, SFO)'], + [], + [], + [] + ], + # empty refinements indicate a primitive action + 'precond': [ + ['At(Home) & Have(Car)'], + ['At(Home)'], + ['At(Home) & Have(Car)'], + ['At(SFOLongTermParking)'], + ['At(Home)'] + ], + 'effect': [ + ['At(SFO) & ~At(Home)'], + ['At(SFO) & ~At(Home)'], + ['At(SFOLongTermParking) & ~At(Home)'], + ['At(SFO) & ~At(SFOLongTermParking)'], + ['At(SFO) & ~At(Home)'] + ]} + """ + indices = [i for i, x in enumerate(library['HLA']) if expr(x).op == self.name] + for i in indices: + actions = [] + for j in range(len(library['steps'][i])): + # find the index of the step [j] of the HLA + index_step = [k for k, x in enumerate(library['HLA']) if x == library['steps'][i][j]][0] + precond = library['precond'][index_step][0] # preconditions of step [j] + effect = library['effect'][index_step][0] # effect of step [j] + actions.append(HLA(library['steps'][i][j], precond, effect)) + yield actions + + def hierarchical_search(self, hierarchy, max_depth=None): + """ + [Figure 11.5] + 'Hierarchical Search, a Breadth First Search implementation of Hierarchical + Forward Planning Search' + The problem is a real-world problem defined by the problem class, and the hierarchy is + a dictionary of HLA - refinements (see refinements generator for details). + With the default max_depth=None the search follows Figure 11.5 exactly and, like the + textbook algorithm, may not terminate on a recursive (cyclic) hierarchy whose goal is + unreachable. Passing max_depth prunes any plan longer than that many actions, which + guarantees termination (returning None when no solution is found within the bound). + """ + act = Node(self.initial, None, [self.actions[0]]) + frontier = deque() + frontier.append(act) + while True: + if not frontier: + return None + plan = frontier.popleft() + # finds the first non primitive hla in plan actions + (hla, index) = RealWorldPlanningProblem.find_hla(plan, hierarchy) + prefix = plan.action[:index] + outcome = RealWorldPlanningProblem( + RealWorldPlanningProblem.result(self.initial, prefix), self.goals, self.actions) + suffix = plan.action[index + 1:] + if not hla: # hla is None and plan is primitive + if outcome.goal_test(): + return plan.action + else: + for sequence in RealWorldPlanningProblem.refinements(hla, hierarchy): # find refinements + refined_plan = prefix + sequence + suffix + if max_depth is None or len(refined_plan) <= max_depth: + frontier.append(Node(outcome.initial, plan, refined_plan)) + + def result(state, actions): + """The outcome of applying an action to the current problem""" + for a in actions: + if a.check_precond(state, a.args): + state = a(state, a.args).clauses + return state + + def angelic_search(self, hierarchy, initial_plan): + """ + [Figure 11.8] + A hierarchical planning algorithm that uses angelic semantics to identify and + commit to high-level plans that work while avoiding high-level plans that don’t. + The predicate MAKING-PROGRESS checks to make sure that we aren’t stuck in an infinite regression + of refinements. + At top level, call ANGELIC-SEARCH with [Act] as the initial_plan. + + InitialPlan contains a sequence of HLA's with angelic semantics + + The possible effects of an angelic HLA in initial_plan are: + ~ : effect remove + $+: effect possibly add + $-: effect possibly remove + $$: possibly add or remove + """ + frontier = deque(initial_plan) + while True: + if not frontier: + return None + plan = frontier.popleft() # sequence of HLA/Angelic HLA's + opt_reachable_set = RealWorldPlanningProblem.reach_opt(self.initial, plan) + pes_reachable_set = RealWorldPlanningProblem.reach_pes(self.initial, plan) + if self.intersects_goal(opt_reachable_set): + if RealWorldPlanningProblem.is_primitive(plan, hierarchy): + return [x for x in plan.action] + guaranteed = self.intersects_goal(pes_reachable_set) + if guaranteed and RealWorldPlanningProblem.making_progress(plan, initial_plan): + final_state = guaranteed[0] # any element of guaranteed + return RealWorldPlanningProblem.decompose(hierarchy, plan, final_state, pes_reachable_set) + # there should be at least one HLA/AngelicHLA, otherwise plan would be primitive + hla, index = RealWorldPlanningProblem.find_hla(plan, hierarchy) + prefix = plan.action[:index] + suffix = plan.action[index + 1:] + outcome = RealWorldPlanningProblem( + RealWorldPlanningProblem.result(self.initial, prefix), self.goals, self.actions) + for sequence in RealWorldPlanningProblem.refinements(hla, hierarchy): # find refinements + frontier.append( + AngelicNode(outcome.initial, plan, prefix + sequence + suffix, prefix + sequence + suffix)) + + def intersects_goal(self, reachable_set): + """ + Find the intersection of the reachable states and the goal + """ + return [y for x in list(reachable_set.keys()) + for y in reachable_set[x] + if all(goal in y for goal in self.goals)] + + def is_primitive(plan, library): + """ + checks if the hla is primitive action + """ + for hla in plan.action: + indices = [i for i, x in enumerate(library['HLA']) if expr(x).op == hla.name] + for i in indices: + if library["steps"][i]: + return False + return True + + def reach_opt(init, plan): + """ + Finds the optimistic reachable set of the sequence of actions in plan + """ + reachable_set = {0: [init]} + optimistic_description = plan.action # list of angelic actions with optimistic description + return RealWorldPlanningProblem.find_reachable_set(reachable_set, optimistic_description) + + def reach_pes(init, plan): + """ + Finds the pessimistic reachable set of the sequence of actions in plan + """ + reachable_set = {0: [init]} + pessimistic_description = plan.action_pes # list of angelic actions with pessimistic description + return RealWorldPlanningProblem.find_reachable_set(reachable_set, pessimistic_description) + + def find_reachable_set(reachable_set, action_description): + """ + Finds the reachable states of the action_description when applied in each state of reachable set. + """ + for i in range(len(action_description)): + reachable_set[i + 1] = [] + if type(action_description[i]) is AngelicHLA: + possible_actions = action_description[i].angelic_action() + else: + possible_actions = action_description + for action in possible_actions: + for state in reachable_set[i]: + if action.check_precond(state, action.args): + if action.effect[0]: + new_state = action(state, action.args).clauses + reachable_set[i + 1].append(new_state) + else: + reachable_set[i + 1].append(state) + return reachable_set + + def find_hla(plan, hierarchy): + """ + Finds the the first HLA action in plan.action, which is not primitive + and its corresponding index in plan.action + """ + hla = None + index = len(plan.action) + for i in range(len(plan.action)): # find the first HLA in plan, that is not primitive + if not RealWorldPlanningProblem.is_primitive(Node(plan.state, plan.parent, [plan.action[i]]), hierarchy): + hla = plan.action[i] + index = i + break + return hla, index + + def making_progress(plan, initial_plan): + """ + Prevents from infinite regression of refinements + + (infinite regression of refinements happens when the algorithm finds a plan that + its pessimistic reachable set intersects the goal inside a call to decompose on + the same plan, in the same circumstances) + """ + for i in range(len(initial_plan)): + if plan == initial_plan[i]: + return False + return True + + def decompose(hierarchy, plan, s_f, reachable_set): + """ + Recursively refine the high-level actions of an abstract plan into a concrete + sequence of primitive actions. Working backwards from the final state ``s_f``, + it picks an intermediate state for each pessimistic action from ``reachable_set`` + and uses angelic search to expand it, returning the assembled primitive solution + (or None if some action cannot be refined). + """ + solution = [] + i = max(reachable_set.keys()) + while plan.action_pes: + action = plan.action_pes.pop() + if i == 0: + return solution + s_i = RealWorldPlanningProblem.find_previous_state(s_f, reachable_set, i, action) + problem = RealWorldPlanningProblem(s_i, s_f, plan.action) + angelic_call = RealWorldPlanningProblem.angelic_search(problem, hierarchy, + [AngelicNode(s_i, Node(None), [action], [action])]) + if angelic_call: + for x in angelic_call: + solution.insert(0, x) + else: + return None + s_f = s_i + i -= 1 + return solution + + def find_previous_state(s_f, reachable_set, i, action): + """ + Given a final state s_f and an action finds a state s_i in reachable_set + such that when action is applied to state s_i returns s_f. + """ + s_i = reachable_set[i - 1][0] + for state in reachable_set[i - 1]: + if s_f in [x for x in RealWorldPlanningProblem.reach_pes( + state, AngelicNode(state, None, [action], [action]))[1]]: + s_i = state + break + return s_i + + +def job_shop_problem(): + """ + [Figure 11.1] JOB-SHOP-PROBLEM + + A job-shop scheduling problem for assembling two cars, + with resource and ordering constraints. + + Example: + >>> from planning import * + >>> p = job_shop_problem() + >>> p.goal_test() + False + >>> p.act(p.jobs[1][0]) + >>> p.act(p.jobs[1][1]) + >>> p.act(p.jobs[1][2]) + >>> p.act(p.jobs[0][0]) + >>> p.act(p.jobs[0][1]) + >>> p.goal_test() + False + >>> p.act(p.jobs[0][2]) + >>> p.goal_test() + True + >>> + """ + resources = {'EngineHoists': 1, 'WheelStations': 2, 'Inspectors': 2, 'LugNuts': 500} + + add_engine1 = HLA('AddEngine1', precond='~Has(C1, E1)', effect='Has(C1, E1)', duration=30, use={'EngineHoists': 1}) + add_engine2 = HLA('AddEngine2', precond='~Has(C2, E2)', effect='Has(C2, E2)', duration=60, use={'EngineHoists': 1}) + add_wheels1 = HLA('AddWheels1', precond='~Has(C1, W1)', effect='Has(C1, W1)', duration=30, use={'WheelStations': 1}, + consume={'LugNuts': 20}) + add_wheels2 = HLA('AddWheels2', precond='~Has(C2, W2)', effect='Has(C2, W2)', duration=15, use={'WheelStations': 1}, + consume={'LugNuts': 20}) + inspect1 = HLA('Inspect1', precond='~Inspected(C1)', effect='Inspected(C1)', duration=10, use={'Inspectors': 1}) + inspect2 = HLA('Inspect2', precond='~Inspected(C2)', effect='Inspected(C2)', duration=10, use={'Inspectors': 1}) + + actions = [add_engine1, add_engine2, add_wheels1, add_wheels2, inspect1, inspect2] + + job_group1 = [add_engine1, add_wheels1, inspect1] + job_group2 = [add_engine2, add_wheels2, inspect2] + + return RealWorldPlanningProblem( + initial='Car(C1) & Car(C2) & Wheels(W1) & Wheels(W2) & Engine(E2) & Engine(E2) & ~Has(C1, E1) & ~Has(C2, ' + 'E2) & ~Has(C1, W1) & ~Has(C2, W2) & ~Inspected(C1) & ~Inspected(C2)', + goals='Has(C1, W1) & Has(C1, E1) & Inspected(C1) & Has(C2, W2) & Has(C2, E2) & Inspected(C2)', + actions=actions, + jobs=[job_group1, job_group2], + resources=resources) + + +def go_to_sfo(): + """Go to SFO Problem""" + + go_home_sfo1 = HLA('Go(Home, SFO)', precond='At(Home) & Have(Car)', effect='At(SFO) & ~At(Home)') + go_home_sfo2 = HLA('Go(Home, SFO)', precond='At(Home)', effect='At(SFO) & ~At(Home)') + drive_home_sfoltp = HLA('Drive(Home, SFOLongTermParking)', precond='At(Home) & Have(Car)', + effect='At(SFOLongTermParking) & ~At(Home)') + shuttle_sfoltp_sfo = HLA('Shuttle(SFOLongTermParking, SFO)', precond='At(SFOLongTermParking)', + effect='At(SFO) & ~At(SFOLongTermParking)') + taxi_home_sfo = HLA('Taxi(Home, SFO)', precond='At(Home)', effect='At(SFO) & ~At(Home)') + + actions = [go_home_sfo1, go_home_sfo2, drive_home_sfoltp, shuttle_sfoltp_sfo, taxi_home_sfo] + + library = { + 'HLA': [ + 'Go(Home, SFO)', + 'Go(Home, SFO)', + 'Drive(Home, SFOLongTermParking)', + 'Shuttle(SFOLongTermParking, SFO)', + 'Taxi(Home, SFO)' + ], + 'steps': [ + ['Drive(Home, SFOLongTermParking)', 'Shuttle(SFOLongTermParking, SFO)'], + ['Taxi(Home, SFO)'], + [], + [], + [] + ], + 'precond': [ + ['At(Home) & Have(Car)'], + ['At(Home)'], + ['At(Home) & Have(Car)'], + ['At(SFOLongTermParking)'], + ['At(Home)'] + ], + 'effect': [ + ['At(SFO) & ~At(Home)'], + ['At(SFO) & ~At(Home)'], + ['At(SFOLongTermParking) & ~At(Home)'], + ['At(SFO) & ~At(SFOLongTermParking)'], + ['At(SFO) & ~At(Home)']]} + + return RealWorldPlanningProblem(initial='At(Home)', goals='At(SFO)', actions=actions), library + + +class AngelicHLA(HLA): + """ + Define Actions for the real-world (that may be refined further), under angelic semantics + """ + + def __init__(self, action, precond, effect, duration=0, consume=None, use=None): + super().__init__(action, precond, effect, duration, consume, use) + + def convert(self, clauses): + """ + Converts strings into Exprs + An HLA with angelic semantics can achieve the effects of simple HLA's (add / remove a variable) + and furthermore can have following effects on the variables:: + + Possibly add variable ( $+ ) + Possibly remove variable ( $- ) + Possibly add or remove a variable ( $$ ) + + Overrides HLA.convert function + """ + lib = {'~': 'Not', + '$+': 'PosYes', + '$-': 'PosNot', + '$$': 'PosYesNot'} + + if isinstance(clauses, Expr): + clauses = conjuncts(clauses) + for i in range(len(clauses)): + for ch in lib.keys(): + if clauses[i].op == ch: + clauses[i] = expr(lib[ch] + str(clauses[i].args[0])) + + elif isinstance(clauses, str): + for ch in lib.keys(): + clauses = clauses.replace(ch, lib[ch]) + if len(clauses) > 0: + clauses = expr(clauses) + + try: + clauses = conjuncts(clauses) + except AttributeError: + pass + + return clauses + + def angelic_action(self): + """ + Converts a high level action (HLA) with angelic semantics into all of its corresponding high level actions (HLA). + An HLA with angelic semantics can achieve the effects of simple HLA's (add / remove a variable) + and furthermore can have following effects for each variable: + + Possibly add variable ( $+: 'PosYes' ) --> corresponds to two HLAs: + HLA_1: add variable + HLA_2: leave variable unchanged + + Possibly remove variable ( $-: 'PosNot' ) --> corresponds to two HLAs: + HLA_1: remove variable + HLA_2: leave variable unchanged + + Possibly add / remove a variable ( $$: 'PosYesNot' ) --> corresponds to three HLAs: + HLA_1: add variable + HLA_2: remove variable + HLA_3: leave variable unchanged + + + example: the angelic action with effects possibly add A and possibly add or remove B corresponds to the + following 6 effects of HLAs: + + + '$+A & $$B': HLA_1: 'A & B' (add A and add B) + HLA_2: 'A & ~B' (add A and remove B) + HLA_3: 'A' (add A) + HLA_4: 'B' (add B) + HLA_5: '~B' (remove B) + HLA_6: ' ' (no effect) + + """ + + effects = [[]] + for clause in self.effect: + (n, w) = AngelicHLA.compute_parameters(clause) + effects = effects * n # create n copies of effects + it = range(1) + if len(effects) != 0: + # split effects into n sublists (separate n copies created in compute_parameters) + it = range(len(effects) // n) + for i in it: + if effects[i]: + if clause.args: + effects[i] = expr(str(effects[i]) + '&' + str( + Expr(clause.op[w:], clause.args[0]))) # make changes in the ith part of effects + if n == 3: + effects[i + len(effects) // 3] = expr( + str(effects[i + len(effects) // 3]) + '&' + str(Expr(clause.op[6:], clause.args[0]))) + else: + effects[i] = expr( + str(effects[i]) + '&' + str(expr(clause.op[w:]))) # make changes in the ith part of effects + if n == 3: + effects[i + len(effects) // 3] = expr( + str(effects[i + len(effects) // 3]) + '&' + str(expr(clause.op[6:]))) + + else: + if clause.args: + effects[i] = Expr(clause.op[w:], clause.args[0]) # make changes in the ith part of effects + if n == 3: + effects[i + len(effects) // 3] = Expr(clause.op[6:], clause.args[0]) + + else: + effects[i] = expr(clause.op[w:]) # make changes in the ith part of effects + if n == 3: + effects[i + len(effects) // 3] = expr(clause.op[6:]) + + return [HLA(Expr(self.name, self.args), self.precond, effects[i]) for i in range(len(effects))] + + def compute_parameters(clause): + """ + computes n,w + + n = number of HLA effects that the angelic HLA corresponds to + w = length of representation of angelic HLA effect + + n = 1, if effect is add + n = 1, if effect is remove + n = 2, if effect is possibly add + n = 2, if effect is possibly remove + n = 3, if effect is possibly add or remove + + """ + if clause.op[:9] == 'PosYesNot': + # possibly add/remove variable: three possible effects for the variable + n = 3 + w = 9 + elif clause.op[:6] == 'PosYes': # possibly add variable: two possible effects for the variable + n = 2 + w = 6 + elif clause.op[:6] == 'PosNot': # possibly remove variable: two possible effects for the variable + n = 2 + w = 3 # We want to keep 'Not' from 'PosNot' when adding action + else: # variable or ~variable + n = 1 + w = 0 + return n, w + + +class AngelicNode(Node): + """ + Extends the class Node. + self.action: contains the optimistic description of an angelic HLA + self.action_pes: contains the pessimistic description of an angelic HLA + """ + + def __init__(self, state, parent=None, action_opt=None, action_pes=None, path_cost=0): + super().__init__(state, parent, action_opt, path_cost) + self.action_pes = action_pes diff --git a/q-learning-9b/aima/probabilistic_learning.py b/q-learning-9b/aima/probabilistic_learning.py new file mode 100644 index 0000000..62b2541 --- /dev/null +++ b/q-learning-9b/aima/probabilistic_learning.py @@ -0,0 +1,158 @@ +"""Learning probabilistic models. (Chapters 20)""" + +import heapq + +from aima.utils import weighted_sampler, product, gaussian + + +class CountingProbDist: + """ + A probability distribution formed by observing and counting examples. + If p is an instance of this class and o is an observed value, then + there are 3 main operations: + p.add(o) increments the count for observation o by 1. + p.sample() returns a random element from the distribution. + p[o] returns the probability for o (as in a regular ProbDist). + """ + + def __init__(self, observations=None, default=0): + """ + Create a distribution, and optionally add in some observations. + By default this is an unsmoothed distribution, but saying default=1, + for example, gives you add-one smoothing. + """ + if observations is None: + observations = [] + self.dictionary = {} + self.n_obs = 0 + self.default = default + self.sampler = None + + for o in observations: + self.add(o) + + def add(self, o): + """Add an observation o to the distribution.""" + self.smooth_for(o) + self.dictionary[o] += 1 + self.n_obs += 1 + self.sampler = None + + def smooth_for(self, o): + """ + Include o among the possible observations, whether or not + it's been observed yet. + """ + if o not in self.dictionary: + self.dictionary[o] = self.default + self.n_obs += self.default + self.sampler = None + + def __getitem__(self, item): + """Return an estimate of the probability of item.""" + self.smooth_for(item) + return self.dictionary[item] / self.n_obs + + # (top() and sample() are not used in this module, but elsewhere.) + + def top(self, n): + """Return (count, obs) tuples for the n most frequent observations.""" + return heapq.nlargest(n, [(v, k) for (k, v) in self.dictionary.items()]) + + def sample(self): + """Return a random sample from the distribution.""" + if self.sampler is None: + self.sampler = weighted_sampler(list(self.dictionary.keys()), list(self.dictionary.values())) + return self.sampler() + + +def NaiveBayesLearner(dataset, continuous=True, simple=False): + """Return a naive Bayes classifier for ``dataset``, dispatching to the simple, + continuous (Gaussian), or discrete variant according to ``simple`` and + ``continuous``.""" + if simple: + return NaiveBayesSimple(dataset) + if continuous: + return NaiveBayesContinuous(dataset) + else: + return NaiveBayesDiscrete(dataset) + + +def NaiveBayesSimple(distribution): + """ + A simple naive bayes classifier that takes as input a dictionary of + CountingProbDist objects and classifies items according to these distributions. + The input dictionary is in the following form:: + + (ClassName, ClassProb): CountingProbDist + """ + target_dist = {c_name: prob for c_name, prob in distribution.keys()} + attr_dists = {c_name: count_prob for (c_name, _), count_prob in distribution.items()} + + def predict(example): + """Predict the target value for example. Calculate probabilities for each + class and pick the max.""" + + def class_probability(target_val): + attr_dist = attr_dists[target_val] + return target_dist[target_val] * product(attr_dist[a] for a in example) + + return max(target_dist.keys(), key=class_probability) + + return predict + + +def NaiveBayesDiscrete(dataset): + """ + Just count how many times each value of each input attribute + occurs, conditional on the target value. Count the different + target values too. + """ + + target_vals = dataset.values[dataset.target] + target_dist = CountingProbDist(target_vals) + attr_dists = {(gv, attr): CountingProbDist(dataset.values[attr]) for gv in target_vals for attr in dataset.inputs} + for example in dataset.examples: + target_val = example[dataset.target] + target_dist.add(target_val) + for attr in dataset.inputs: + attr_dists[target_val, attr].add(example[attr]) + + def predict(example): + """ + Predict the target value for example. Consider each possible value, + and pick the most likely by looking at each attribute independently. + """ + + def class_probability(target_val): + return (target_dist[target_val] * product(attr_dists[target_val, attr][example[attr]] + for attr in dataset.inputs)) + + return max(target_vals, key=class_probability) + + return predict + + +def NaiveBayesContinuous(dataset): + """ + Count how many times each target value occurs. + Also, find the means and deviations of input attribute values for each target value. + """ + means, deviations = dataset.find_means_and_deviations() + + target_vals = dataset.values[dataset.target] + target_dist = CountingProbDist(target_vals) + + def predict(example): + """Predict the target value for example. Consider each possible value, + and pick the most likely by looking at each attribute independently.""" + + def class_probability(target_val): + prob = target_dist[target_val] + for attr in dataset.inputs: + prob *= gaussian(means[target_val][attr], deviations[target_val][attr], example[attr]) + return prob + + return max(target_vals, key=class_probability) + + return predict diff --git a/q-learning-9b/aima/probability.py b/q-learning-9b/aima/probability.py new file mode 100644 index 0000000..07a8655 --- /dev/null +++ b/q-learning-9b/aima/probability.py @@ -0,0 +1,1389 @@ +"""Probability models (Chapter 13-15)""" + +import copy +import math +import re +from collections import defaultdict +from functools import reduce + +from aima.agents import Agent +from aima.utils import * + + +def DTAgentProgram(belief_state): + """ + [Figure 13.1] + A decision-theoretic agent. + """ + + def program(percept): + belief_state.observe(program.action, percept) + program.action = max(belief_state.actions(), key=belief_state.expected_outcome_utility) + return program.action + + program.action = None + return program + + +# ______________________________________________________________________________ + + +class ProbDist: + """A discrete probability distribution. You name the random variable + in the constructor, then assign and query probability of values. + >>> P = ProbDist('Flip'); P['H'], P['T'] = 0.25, 0.75; P['H'] + 0.25 + >>> P = ProbDist('X', {'lo': 125, 'med': 375, 'hi': 500}) + >>> P['lo'], P['med'], P['hi'] + (0.125, 0.375, 0.5) + """ + + def __init__(self, var_name='?', freq=None): + """If freq is given, it is a dictionary of values - frequency pairs, + then ProbDist is normalized.""" + self.prob = {} + self.var_name = var_name + self.values = [] + if freq: + for (v, p) in freq.items(): + self[v] = p + self.normalize() + + def __getitem__(self, val): + """Given a value, return P(value).""" + try: + return self.prob[val] + except KeyError: + return 0 + + def __setitem__(self, val, p): + """Set P(val) = p.""" + if val not in self.values: + self.values.append(val) + self.prob[val] = p + + def normalize(self): + """Make sure the probabilities of all values sum to 1. + Returns the normalized distribution. + Raises a ZeroDivisionError if the sum of the values is 0.""" + total = sum(self.prob.values()) + if not np.isclose(total, 1.0): + for val in self.prob: + self.prob[val] /= total + return self + + def show_approx(self, numfmt='{:.3g}'): + """Show the probabilities rounded and sorted by key, for the + sake of portable doctests.""" + return ', '.join([('{}: ' + numfmt).format(v, p) for (v, p) in sorted(self.prob.items())]) + + def __repr__(self): + return "P({})".format(self.var_name) + + +class JointProbDist(ProbDist): + """A discrete probability distribute over a set of variables. + >>> P = JointProbDist(['X', 'Y']); P[1, 1] = 0.25 + >>> P[1, 1] + 0.25 + >>> P[dict(X=0, Y=1)] = 0.5 + >>> P[dict(X=0, Y=1)] + 0.5""" + + def __init__(self, variables): + self.prob = {} + self.variables = variables + self.vals = defaultdict(list) + + def __getitem__(self, values): + """Given a tuple or dict of values, return P(values).""" + values = event_values(values, self.variables) + return ProbDist.__getitem__(self, values) + + def __setitem__(self, values, p): + """Set P(values) = p. Values can be a tuple or a dict; it must + have a value for each of the variables in the joint. Also keep track + of the values we have seen so far for each variable.""" + values = event_values(values, self.variables) + self.prob[values] = p + for var, val in zip(self.variables, values): + if val not in self.vals[var]: + self.vals[var].append(val) + + def values(self, var): + """Return the set of possible values for a variable.""" + return self.vals[var] + + def __repr__(self): + return "P({})".format(self.variables) + + +def event_values(event, variables): + """Return a tuple of the values of variables in event. + >>> event_values ({'A': 10, 'B': 9, 'C': 8}, ['C', 'A']) + (8, 10) + >>> event_values ((1, 2), ['C', 'A']) + (1, 2) + """ + if isinstance(event, tuple) and len(event) == len(variables): + return event + else: + return tuple([event[var] for var in variables]) + + +# ______________________________________________________________________________ + + +def enumerate_joint_ask(X, e, P): + """ + [Section 13.3] + Return a probability distribution over the values of the variable X, + given the {var:val} observations e, in the JointProbDist P. + >>> P = JointProbDist(['X', 'Y']) + >>> P[0,0] = 0.25; P[0,1] = 0.5; P[1,1] = P[2,1] = 0.125 + >>> enumerate_joint_ask('X', dict(Y=1), P).show_approx() + '0: 0.667, 1: 0.167, 2: 0.167' + """ + assert X not in e, "Query variable must be distinct from evidence" + Q = ProbDist(X) # probability distribution for X, initially empty + Y = [v for v in P.variables if v != X and v not in e] # hidden variables. + for xi in P.values(X): + Q[xi] = enumerate_joint(Y, extend(e, X, xi), P) + return Q.normalize() + + +def enumerate_joint(variables, e, P): + """Return the sum of those entries in P consistent with e, + provided variables is P's remaining variables (the ones not in e).""" + if not variables: + return P[e] + Y, rest = variables[0], variables[1:] + return sum([enumerate_joint(rest, extend(e, Y, y), P) for y in P.values(Y)]) + + +# ______________________________________________________________________________ +# Independence + + +def is_independent(variables, P): + """ + Return whether a list of variables are independent given their distribution P + P is an instance of JoinProbDist + >>> P = JointProbDist(['X', 'Y']) + >>> P[0,0] = 0.25; P[0,1] = 0.5; P[1,1] = P[1,0] = 0.125 + >>> is_independent(['X', 'Y'], P) + False + """ + for var in variables: + event_vars = variables[:] + event_vars.remove(var) + event = {} + distribution = enumerate_joint_ask(var, event, P) + events = gen_possible_events(event_vars, P) + for e in events: + conditional_distr = enumerate_joint_ask(var, e, P) + if conditional_distr.prob != distribution.prob: + return False + return True + + +def gen_possible_events(vars, P): + """Generate all possible events of a collection of vars according to distribution of P""" + events = [] + + def backtrack(vars, P, temp): + if not vars: + events.append(temp) + return + var = vars[0] + for val in P.values(var): + temp[var] = val + backtrack([v for v in vars if v != var], P, copy.copy(temp)) + + backtrack(vars, P, {}) + return events + + +# ______________________________________________________________________________ + + +class BayesNet: + """Bayesian network containing only boolean-variable nodes.""" + + def __init__(self, node_specs=None): + """Nodes must be ordered with parents before children.""" + self.nodes = [] + self.variables = [] + node_specs = node_specs or [] + for node_spec in node_specs: + self.add(node_spec) + + def add(self, node_spec): + """Add a node to the net. Its parents must already be in the + net, and its variable must not.""" + node = BayesNode(*node_spec) + assert node.variable not in self.variables + assert all((parent in self.variables) for parent in node.parents) + self.nodes.append(node) + self.variables.append(node.variable) + for parent in node.parents: + self.variable_node(parent).children.append(node) + + def variable_node(self, var): + """Return the node for the variable named var. + >>> burglary.variable_node('Burglary').variable + 'Burglary'""" + for n in self.nodes: + if n.variable == var: + return n + raise Exception("No such variable: {}".format(var)) + + def variable_values(self, var): + """Return the domain of var.""" + return [True, False] + + def __repr__(self): + return 'BayesNet({0!r})'.format(self.nodes) + + +class DecisionNetwork(BayesNet): + """An abstract class for a decision network as a wrapper for a BayesNet. + Represents an agent's current state, its possible actions, reachable states + and utilities of those states.""" + + def __init__(self, action, infer): + """action: a single action node + infer: the preferred method to carry out inference on the given BayesNet""" + super(DecisionNetwork, self).__init__() + self.action = action + self.infer = infer + + def best_action(self): + """Return the best action in the network""" + return self.action + + def get_utility(self, action, state): + """Return the utility for a particular action and state in the network""" + raise NotImplementedError + + def get_expected_utility(self, action, evidence): + """Compute the expected utility given an action and evidence""" + u = 0.0 + prob_dist = self.infer(action, evidence, self).prob + for item, _ in prob_dist.items(): + u += prob_dist[item] * self.get_utility(action, item) + + return u + + +class InformationGatheringAgent(Agent): + """ + [Figure 16.9] + A simple information gathering agent. The agent works by repeatedly selecting + the observation with the highest information value, until the cost of the next + observation is greater than its expected benefit.""" + + def __init__(self, decnet, infer, initial_evidence=None): + """decnet: a decision network + infer: the preferred method to carry out inference on the given decision network + initial_evidence: initial evidence""" + self.decnet = decnet + self.infer = infer + self.observation = initial_evidence or [] + self.variables = self.decnet.nodes + + def integrate_percept(self, percept): + """Integrate the given percept into the decision network""" + raise NotImplementedError + + def execute(self, percept): + """Execute the information gathering algorithm""" + self.observation = self.integrate_percept(percept) + vpis = self.vpi_cost_ratio(self.variables) + j = max(vpis) + variable = self.variables[j] + + if self.vpi(variable) > self.cost(variable): + return self.request(variable) + + return self.decnet.best_action() + + def request(self, variable): + """Return the value of the given random variable as the next percept""" + raise NotImplementedError + + def cost(self, var): + """Return the cost of obtaining evidence through tests, consultants or questions""" + raise NotImplementedError + + def vpi_cost_ratio(self, variables): + """Return the VPI to cost ratio for the given variables""" + v_by_c = [] + for var in variables: + v_by_c.append(self.vpi(var) / self.cost(var)) + return v_by_c + + def vpi(self, variable): + """Return VPI for a given variable""" + vpi = 0.0 + prob_dist = self.infer(variable, self.observation, self.decnet).prob + for item, _ in prob_dist.items(): + post_prob = prob_dist[item] + new_observation = list(self.observation) + new_observation.append(item) + expected_utility = self.decnet.get_expected_utility(variable, new_observation) + vpi += post_prob * expected_utility + + vpi -= self.decnet.get_expected_utility(variable, self.observation) + return vpi + + +class BayesNode: + """A conditional probability distribution for a boolean variable, + P(X | parents). Part of a BayesNet.""" + + def __init__(self, X, parents, cpt): + """X is a variable name, and parents a sequence of variable + names or a space-separated string. cpt, the conditional + probability table, takes one of these forms: + + * A number, the unconditional probability P(X=true). You can + use this form when there are no parents. + + * A dict {v: p, ...}, the conditional probability distribution + P(X=true | parent=v) = p. When there's just one parent. + + * A dict {(v1, v2, ...): p, ...}, the distribution P(X=true | + parent1=v1, parent2=v2, ...) = p. Each key must have as many + values as there are parents. You can use this form always; + the first two are just conveniences. + + In all cases the probability of X being false is left implicit, + since it follows from P(X=true). + + >>> X = BayesNode('X', '', 0.2) + >>> Y = BayesNode('Y', 'P', {T: 0.2, F: 0.7}) + >>> Z = BayesNode('Z', 'P Q', + ... {(T, T): 0.2, (T, F): 0.3, (F, T): 0.5, (F, F): 0.7}) + """ + if isinstance(parents, str): + parents = parents.split() + + # We store the table always in the third form above. + if isinstance(cpt, (float, int)): # no parents, 0-tuple + cpt = {(): cpt} + elif isinstance(cpt, dict): + # one parent, 1-tuple + if cpt and isinstance(list(cpt.keys())[0], bool): + cpt = {(v,): p for v, p in cpt.items()} + + assert isinstance(cpt, dict) + for vs, p in cpt.items(): + assert isinstance(vs, tuple) and len(vs) == len(parents) + assert all(isinstance(v, bool) for v in vs) + assert 0 <= p <= 1 + + self.variable = X + self.parents = parents + self.cpt = cpt + self.children = [] + + def p(self, value, event): + """Return the conditional probability + P(X=value | parents=parent_values), where parent_values + are the values of parents in event. (event must assign each + parent a value.) + >>> bn = BayesNode('X', 'Burglary', {T: 0.2, F: 0.625}) + >>> bn.p(False, {'Burglary': False, 'Earthquake': True}) + 0.375""" + assert isinstance(value, bool) + ptrue = self.cpt[event_values(event, self.parents)] + return ptrue if value else 1 - ptrue + + def sample(self, event): + """Sample from the distribution for this variable conditioned + on event's values for parent_variables. That is, return True/False + at random according with the conditional probability given the + parents.""" + return probability(self.p(True, event)) + + def __repr__(self): + return repr((self.variable, ' '.join(self.parents))) + + +class DiscreteBayesNode: + """A node of a discrete Bayesian network whose variable may take more than two + values (unlike :class:`BayesNode`, which is boolean). + + ``values`` is the variable's domain. ``cpt`` maps each tuple of parent values + (ordered as in ``parents``) to the probabilities over ``values`` -- either a + sequence in domain order or a ``{value: prob}`` dict. A root node uses the + empty tuple ``()`` as its only key. + """ + + def __init__(self, X, parents, values, cpt): + if isinstance(parents, str): + parents = parents.split() + self.variable = X + self.parents = parents + self.values = list(values) + self.cpt = {} + for key, dist in cpt.items(): + key = key if isinstance(key, tuple) else (key,) + assert len(key) == len(self.parents) + self.cpt[key] = dist if isinstance(dist, dict) else dict(zip(self.values, dist)) + self.children = [] + + def p(self, value, event): + """Return P(X = ``value`` | parents = their values in ``event``).""" + return self.cpt[tuple(event[parent] for parent in self.parents)][value] + + def __repr__(self): + return repr((self.variable, ' '.join(self.parents))) + + +class DiscreteBayesNet: + """A Bayesian network of :class:`DiscreteBayesNode`\\ s (variables with arbitrary + finite domains). Exact inference works through the generic + :func:`enumeration_ask` / :func:`elimination_ask`, which rely only on + ``node.p`` and ``variable_values`` and so work unchanged for multi-valued nodes. + """ + + def __init__(self, node_specs=None): + self.nodes = [] + self.variables = [] + for spec in node_specs or []: + self.add(spec) + + def add(self, node_spec): + """Add a ``DiscreteBayesNode`` (or a ``(name, parents, values, cpt)`` spec); + its parents must already be in the net and its variable must not.""" + node = node_spec if isinstance(node_spec, DiscreteBayesNode) else DiscreteBayesNode(*node_spec) + assert node.variable not in self.variables + assert all(parent in self.variables for parent in node.parents) + self.nodes.append(node) + self.variables.append(node.variable) + for parent in node.parents: + self.variable_node(parent).children.append(node) + + def variable_node(self, var): + for n in self.nodes: + if n.variable == var: + return n + raise Exception("No such variable: {}".format(var)) + + def variable_values(self, var): + """Return the domain of var.""" + return self.variable_node(var).values + + def __repr__(self): + return 'DiscreteBayesNet({0!r})'.format(self.nodes) + + +def read_bif(source): + """Parse a Bayesian network in BIF (Bayesian Interchange Format) into a + :class:`DiscreteBayesNet`. ``source`` is the BIF text or an open file object. + + BIF is the format used by the Bayesian Network Repository + (https://www.bnlearn.com/bnrepository/), so this lets aima load standard + multi-valued networks such as the car-insurance ("Insurance") model. + """ + text = source.read() if hasattr(source, 'read') else source + + # variable NAME { type discrete [ k ] { v1, v2, ... }; } + domains = {name: [v.strip() for v in vals.split(',') if v.strip()] + for name, vals in re.findall(r'variable\s+(\w+)\s*\{[^}]*?\{([^}]*)\}', text)} + + # probability ( VAR [ | P1, P2, ... ] ) { table ... ; | (pv, ...) p, ... ; } + specs = {} + for header, body in re.findall(r'probability\s*\(\s*([^)]*?)\s*\)\s*\{(.*?)\}', text, re.S): + if '|' in header: + var, parent_str = header.split('|') + var, parents = var.strip(), [p.strip() for p in parent_str.split(',') if p.strip()] + else: + var, parents = header.strip(), [] + cpt = {} + table = re.search(r'table\s+([^;]+);', body) + if table: + cpt[()] = [float(x) for x in table.group(1).split(',')] + else: + for key, probs in re.findall(r'\(([^)]*)\)\s*([^;]+);', body): + cpt[tuple(v.strip() for v in key.split(',') if v.strip())] = \ + [float(x) for x in probs.split(',')] + specs[var] = (parents, domains[var], cpt) + + # add nodes parents-before-children (the BIF order need not be topological) + net, added = DiscreteBayesNet(), set() + + def add_node(name): + if name in added: + return + parents, values, cpt = specs[name] + for parent in parents: + add_node(parent) + net.add(DiscreteBayesNode(name, parents, values, cpt)) + added.add(name) + + for name in specs: + add_node(name) + return net + + +def insurance(): + """Return the car-insurance ("Insurance") Bayesian network as a + :class:`DiscreteBayesNet`, loaded from ``aima-data/insurance.bif``. + + This is the 27-variable discrete model of Binder, Koller, Russell & Kanazawa + (1997) referenced by the AIMA 4e car-insurance case study (Section 16); the + book notes the discrete conditional distributions are provided in the code + repository, and this is them. + """ + return read_bif(open_data('insurance.bif').read()) + + +# Burglary example [Figure 14.2] + +T, F = True, False + +burglary = BayesNet([('Burglary', '', 0.001), + ('Earthquake', '', 0.002), + ('Alarm', 'Burglary Earthquake', + {(T, T): 0.95, (T, F): 0.94, (F, T): 0.29, (F, F): 0.001}), + ('JohnCalls', 'Alarm', {T: 0.90, F: 0.05}), + ('MaryCalls', 'Alarm', {T: 0.70, F: 0.01})]) + + +# ______________________________________________________________________________ +# Bayesian nets with continuous variables + + +def gaussian_probability(param, event, value): + """ + Gaussian probability of a continuous Bayesian network node on condition of + certain event and the parameters determined by the event + + :param param: parameters determined by discrete parent events of current node + :param event: a dict, continuous event of current node, the values are used + as parameters in calculating distribution + :param value: float, the value of current continuous node + :return: float, the calculated probability + + >>> param = {'sigma':0.5, 'b':1, 'a':{'h1':0.5, 'h2': 1.5}} + >>> event = {'h1':0.6, 'h2': 0.3} + >>> gaussian_probability(param, event, 1) + 0.2590351913317835 + """ + + assert isinstance(event, dict) + assert isinstance(param, dict) + buff = 0 + for k, v in event.items(): + # buffer varianle to calculate h1*a_h1 + h2*a_h2 + buff += param['a'][k] * v + res = 1 / (param['sigma'] * np.sqrt(2 * np.pi)) * np.exp(-0.5 * ((value - buff - param['b']) / param['sigma']) ** 2) + return res + + +def logistic_probability(param, event, value): + """ + Logistic probability of a discrete node in Bayesian network with continuous parents, + :param param: a dict, parameters determined by discrete parents of current node + :param event: a dict, names and values of continuous parent variables of current node + :param value: boolean, True or False + :return: int, probability + """ + + buff = 1 + for _, v in event.items(): + # buffer variable to calculate (value-mu)/sigma + + buff *= (v - param['mu']) / param['sigma'] + p = 1 - 1 / (1 + np.exp(-4 / np.sqrt(2 * np.pi) * buff)) + return p if value else 1 - p + + +class ContinuousBayesNode: + """ A Bayesian network node with continuous distribution or with continuous distributed parents """ + + def __init__(self, name, d_parents, c_parents, parameters, type): + """ + A continuous Bayesian node has two types of parents: discrete and continuous. + :param d_parents: str, name of discrete parents, value of which determines distribution parameters + :param c_parents: str, name of continuous parents, value of which is used to calculate distribution + :param parameters: a dict, parameters for distribution of current node, keys corresponds to discrete parents + :param type: str, type of current node's value, either 'd' (discrete) or 'c'(continuous) + """ + + self.parameters = parameters + self.type = type + self.d_parents = d_parents.split() + self.c_parents = c_parents.split() + self.parents = self.d_parents + self.c_parents + self.variable = name + self.children = [] + + def continuous_p(self, value, c_event, d_event): + """ + Probability given the value of current node and its parents + :param c_event: event of continuous nodes + :param d_event: event of discrete nodes + """ + assert isinstance(c_event, dict) + assert isinstance(d_event, dict) + + d_event_vals = event_values(d_event, self.d_parents) + if len(d_event_vals) == 1: + d_event_vals = d_event_vals[0] + param = self.parameters[d_event_vals] + if self.type == "c": + p = gaussian_probability(param, c_event, value) + if self.type == "d": + p = logistic_probability(param, c_event, value) + return p + + +# ______________________________________________________________________________ + + +def enumeration_ask(X, e, bn): + """ + [Figure 14.9] + Return the conditional probability distribution of variable X + given evidence e, from BayesNet bn. + >>> enumeration_ask('Burglary', dict(JohnCalls=T, MaryCalls=T), burglary + ... ).show_approx() + 'False: 0.716, True: 0.284'""" + assert X not in e, "Query variable must be distinct from evidence" + Q = ProbDist(X) + for xi in bn.variable_values(X): + Q[xi] = enumerate_all(bn.variables, extend(e, X, xi), bn) + return Q.normalize() + + +def enumerate_all(variables, e, bn): + """Return the sum of those entries in P(variables | e{others}) + consistent with e, where P is the joint distribution represented + by bn, and e{others} means e restricted to bn's other variables + (the ones other than variables). Parents must precede children in variables.""" + if not variables: + return 1.0 + Y, rest = variables[0], variables[1:] + Ynode = bn.variable_node(Y) + if Y in e: + return Ynode.p(e[Y], e) * enumerate_all(rest, e, bn) + else: + return sum(Ynode.p(y, e) * enumerate_all(rest, extend(e, Y, y), bn) + for y in bn.variable_values(Y)) + + +# ______________________________________________________________________________ + + +def elimination_ask(X, e, bn): + """ + [Figure 14.11] + Compute bn's P(X|e) by variable elimination. + >>> elimination_ask('Burglary', dict(JohnCalls=T, MaryCalls=T), burglary + ... ).show_approx() + 'False: 0.716, True: 0.284'""" + assert X not in e, "Query variable must be distinct from evidence" + factors = [] + for var in reversed(bn.variables): + factors.append(make_factor(var, e, bn)) + if is_hidden(var, X, e): + factors = sum_out(var, factors, bn) + return pointwise_product(factors, bn).normalize() + + +def is_hidden(var, X, e): + """Is var a hidden variable when querying P(X|e)?""" + return var != X and var not in e + + +def make_factor(var, e, bn): + """Return the factor for var in bn's joint distribution given e. + That is, bn's full joint distribution, projected to accord with e, + is the pointwise product of these factors for bn's variables.""" + node = bn.variable_node(var) + variables = [X for X in [var] + node.parents if X not in e] + cpt = {event_values(e1, variables): node.p(e1[var], e1) + for e1 in all_events(variables, bn, e)} + return Factor(variables, cpt) + + +def pointwise_product(factors, bn): + """Multiply a sequence of factors together into a single factor over the union of their + variables, using the Bayes net ``bn`` to enumerate variable values.""" + return reduce(lambda f, g: f.pointwise_product(g, bn), factors) + + +def sum_out(var, factors, bn): + """Eliminate var from all factors by summing over its values.""" + result, var_factors = [], [] + for f in factors: + (var_factors if var in f.variables else result).append(f) + result.append(pointwise_product(var_factors, bn).sum_out(var, bn)) + return result + + +class Factor: + """A factor in a joint distribution.""" + + def __init__(self, variables, cpt): + self.variables = variables + self.cpt = cpt + + def pointwise_product(self, other, bn): + """Multiply two factors, combining their variables.""" + variables = list(set(self.variables) | set(other.variables)) + cpt = {event_values(e, variables): self.p(e) * other.p(e) for e in all_events(variables, bn, {})} + return Factor(variables, cpt) + + def sum_out(self, var, bn): + """Make a factor eliminating var by summing over its values.""" + variables = [X for X in self.variables if X != var] + cpt = {event_values(e, variables): sum(self.p(extend(e, var, val)) for val in bn.variable_values(var)) + for e in all_events(variables, bn, {})} + return Factor(variables, cpt) + + def normalize(self): + """Return my probabilities; must be down to one variable.""" + assert len(self.variables) == 1 + return ProbDist(self.variables[0], {k: v for ((k,), v) in self.cpt.items()}) + + def p(self, e): + """Look up my value tabulated for e.""" + return self.cpt[event_values(e, self.variables)] + + +def all_events(variables, bn, e): + """Yield every way of extending e with values for all variables.""" + if not variables: + yield e + else: + X, rest = variables[0], variables[1:] + for e1 in all_events(rest, bn, e): + for x in bn.variable_values(X): + yield extend(e1, X, x) + + +# ______________________________________________________________________________ + +# [Figure 14.12a]: sprinkler network + + +sprinkler = BayesNet([('Cloudy', '', 0.5), + ('Sprinkler', 'Cloudy', {T: 0.10, F: 0.50}), + ('Rain', 'Cloudy', {T: 0.80, F: 0.20}), + ('WetGrass', 'Sprinkler Rain', + {(T, T): 0.99, (T, F): 0.90, (F, T): 0.90, (F, F): 0.00})]) + + +# ______________________________________________________________________________ + + +def prior_sample(bn): + """ + [Figure 14.13] + Randomly sample from bn's full joint distribution. + The result is a {variable: value} dict. + """ + event = {} + for node in bn.nodes: + event[node.variable] = node.sample(event) + return event + + +# _________________________________________________________________________ + + +def rejection_sampling(X, e, bn, N=10000): + """ + [Figure 14.14] + Estimate the probability distribution of variable X given + evidence e in BayesNet bn, using N samples. + Raises a ZeroDivisionError if all the N samples are rejected, + i.e., inconsistent with e. + >>> random.seed(47) + >>> rejection_sampling('Burglary', dict(JohnCalls=T, MaryCalls=T), + ... burglary, 10000).show_approx() + 'False: 0.7, True: 0.3' + """ + counts = {x: 0 for x in bn.variable_values(X)} # bold N in [Figure 14.14] + for j in range(N): + sample = prior_sample(bn) # boldface x in [Figure 14.14] + if consistent_with(sample, e): + counts[sample[X]] += 1 + return ProbDist(X, counts) + + +def consistent_with(event, evidence): + """Is event consistent with the given evidence?""" + return all(evidence.get(k, v) == v for k, v in event.items()) + + +# _________________________________________________________________________ + + +def likelihood_weighting(X, e, bn, N=10000): + """ + [Figure 14.15] + Estimate the probability distribution of variable X given + evidence e in BayesNet bn. + >>> random.seed(1017) + >>> likelihood_weighting('Burglary', dict(JohnCalls=T, MaryCalls=T), + ... burglary, 10000).show_approx() + 'False: 0.702, True: 0.298' + """ + W = {x: 0 for x in bn.variable_values(X)} + for j in range(N): + sample, weight = weighted_sample(bn, e) # boldface x, w in [Figure 14.15] + W[sample[X]] += weight + return ProbDist(X, W) + + +def weighted_sample(bn, e): + """ + Sample an event from bn that's consistent with the evidence e; + return the event and its weight, the likelihood that the event + accords to the evidence. + """ + w = 1 + event = dict(e) # boldface x in [Figure 14.15] + for node in bn.nodes: + Xi = node.variable + if Xi in e: + w *= node.p(e[Xi], event) + else: + event[Xi] = node.sample(event) + return event, w + + +# _________________________________________________________________________ + + +def gibbs_ask(X, e, bn, N=1000): + """[Figure 14.16] Approximate P(X | e) by Gibbs sampling: from a random state + consistent with the evidence, repeatedly resample each nonevidence variable from + its Markov blanket and tally how often X takes each value.""" + assert X not in e, "Query variable must be distinct from evidence" + counts = {x: 0 for x in bn.variable_values(X)} # bold N in [Figure 14.16] + Z = [var for var in bn.variables if var not in e] + state = dict(e) # boldface x in [Figure 14.16] + for Zi in Z: + state[Zi] = random.choice(bn.variable_values(Zi)) + for j in range(N): + for Zi in Z: + state[Zi] = markov_blanket_sample(Zi, state, bn) + counts[state[X]] += 1 + return ProbDist(X, counts) + + +def markov_blanket_sample(X, e, bn): + """Return a sample from P(X | mb) where mb denotes that the + variables in the Markov blanket of X take their values from event + e (which must assign a value to each). The Markov blanket of X is + X's parents, children, and children's parents.""" + Xnode = bn.variable_node(X) + Q = ProbDist(X) + for xi in bn.variable_values(X): + ei = extend(e, X, xi) + # [Equation 14.12] + Q[xi] = Xnode.p(xi, e) * product(Yj.p(ei[Yj.variable], ei) for Yj in Xnode.children) + # (assuming a Boolean variable here) + return probability(Q.normalize()[True]) + + +# _________________________________________________________________________ + + +class HiddenMarkovModel: + """A Hidden markov model which takes Transition model and Sensor model as inputs""" + + def __init__(self, transition_model, sensor_model, prior=None): + self.transition_model = transition_model + self.sensor_model = sensor_model + self.prior = prior or [0.5, 0.5] + + def sensor_dist(self, ev): + """Return the sensor (observation) distribution corresponding to the evidence ``ev``: + the first row of the sensor model when ``ev`` is True, otherwise the second.""" + if ev is True: + return self.sensor_model[0] + else: + return self.sensor_model[1] + + +def forward(HMM, fv, ev): + """Perform one forward (filtering) step of an HMM: project the forward message ``fv`` + through the transition model, weight it by the sensor distribution for evidence ``ev``, + and return the normalized next forward message. [Figure 15.4]""" + prediction = vector_add(scalar_vector_product(fv[0], HMM.transition_model[0]), + scalar_vector_product(fv[1], HMM.transition_model[1])) + sensor_dist = HMM.sensor_dist(ev) + + return normalize(element_wise_product(sensor_dist, prediction)) + + +def backward(HMM, b, ev): + """Perform one backward step of an HMM: weight the backward message ``b`` by the sensor + distribution for evidence ``ev`` and propagate it through the transition model, returning + the normalized previous backward message. [Figure 15.4]""" + sensor_dist = HMM.sensor_dist(ev) + prediction = element_wise_product(sensor_dist, b) + + return normalize(vector_add(scalar_vector_product(prediction[0], HMM.transition_model[0]), + scalar_vector_product(prediction[1], HMM.transition_model[1]))) + + +def forward_backward(HMM, ev): + """ + [Figure 15.4] + Forward-Backward algorithm for smoothing. Computes posterior probabilities + of a sequence of states given a sequence of observations. + """ + t = len(ev) + ev.insert(0, None) # to make the code look similar to pseudo code + + fv = [[0.0, 0.0] for _ in range(len(ev))] + b = [1.0, 1.0] + sv = [[0, 0] for _ in range(len(ev))] + + fv[0] = HMM.prior + + for i in range(1, t + 1): + fv[i] = forward(HMM, fv[i - 1], ev[i]) + for i in range(t, -1, -1): + sv[i - 1] = normalize(element_wise_product(fv[i], b)) + b = backward(HMM, b, ev[i]) + + sv = sv[::-1] + + return sv + + +def viterbi(HMM, ev): + """ + [Equation 15.11] + Viterbi algorithm to find the most likely sequence. Computes the best path and the + corresponding probabilities, given an HMM model and a sequence of observations. + """ + t = len(ev) + ev = ev.copy() + ev.insert(0, None) + + m = [[0.0, 0.0] for _ in range(len(ev) - 1)] + + # the recursion is initialized with m1 = forward(P(X0), e1) + m[0] = forward(HMM, HMM.prior, ev[1]) + # keep track of maximizing predecessors + backtracking_graph = [] + + for i in range(1, t): + m[i] = element_wise_product(HMM.sensor_dist(ev[i + 1]), + [max(element_wise_product(HMM.transition_model[0], m[i - 1])), + max(element_wise_product(HMM.transition_model[1], m[i - 1]))]) + backtracking_graph.append([np.argmax(element_wise_product(HMM.transition_model[0], m[i - 1])), + np.argmax(element_wise_product(HMM.transition_model[1], m[i - 1]))]) + + # computed probabilities + ml_probabilities = [0.0] * (len(ev) - 1) + # most likely sequence + ml_path = [True] * (len(ev) - 1) + + # the construction of the most likely sequence starts in the final state with the largest probability, and + # runs backwards; the algorithm needs to store for each xt its predecessor xt-1 maximizing its probability + i_max = np.argmax(m[-1]) + + for i in range(t - 1, -1, -1): + ml_probabilities[i] = m[i][i_max] + ml_path[i] = True if i_max == 0 else False + if i > 0: + i_max = backtracking_graph[i - 1][i_max] + + return ml_path, ml_probabilities + + +def baum_welch(HMM, observations, iterations=100): + """ + [Section 20.3] + Baum-Welch algorithm: the instance of EM that learns the parameters of a + Hidden Markov Model (transition model, sensor model and prior) from a single + sequence of boolean 'observations', starting from the initial guess in 'HMM'. + Each iteration runs a (scaled) forward-backward pass to compute the smoothed + state marginals gamma_t(i) = P(X_t=i | e_1:T) and transition marginals + xi_t(i,j) = P(X_t=i, X_t+1=j | e_1:T) (E-step), then re-estimates every + parameter as the corresponding normalized expected count (M-step):: + + prior_i = gamma_0(i) + A_ij = sum_t xi_t(i, j) / sum_t gamma_t(i) + sensor_oi = sum_{t: e_t = o} gamma_t(i) / sum_t gamma_t(i) + + Returns a new HiddenMarkovModel with the learned parameters. + """ + A = np.array(HMM.transition_model, dtype=float) + prior = np.array(HMM.prior, dtype=float) + # sensor[0] = P(e=True | state), sensor[1] = P(e=False | state) + sensor = np.array(HMM.sensor_model, dtype=float) + obs = list(observations) + n, t_max = len(prior), len(obs) + + for _ in range(iterations): + # emission vectors b_t(i) = P(e_t | X_t = i), recomputed from current sensor + B = np.array([sensor[0] if e else sensor[1] for e in obs]) + + # E-step: scaled forward (alpha) and backward (beta) messages + alpha, c = np.zeros((t_max, n)), np.zeros(t_max) + alpha[0] = prior * B[0] + c[0] = alpha[0].sum() + alpha[0] /= c[0] + for t in range(1, t_max): + alpha[t] = B[t] * (alpha[t - 1] @ A) + c[t] = alpha[t].sum() + alpha[t] /= c[t] + beta = np.zeros((t_max, n)) + beta[-1] = 1 + for t in range(t_max - 2, -1, -1): + beta[t] = (A @ (B[t + 1] * beta[t + 1])) / c[t + 1] + + # smoothed state and transition marginals (normalized, so the per-step + # scaling factors cancel out) + gamma = alpha * beta + gamma /= gamma.sum(axis=1, keepdims=True) + xi = np.zeros((t_max - 1, n, n)) + for t in range(t_max - 1): + xi[t] = alpha[t][:, None] * A * B[t + 1] * beta[t + 1] + xi[t] /= xi[t].sum() + + # M-step: re-estimate every parameter from the expected counts + prior = gamma[0] + A = xi.sum(axis=0) / gamma[:-1].sum(axis=0)[:, None] + mask = np.array(obs, dtype=bool) + p_true = gamma[mask].sum(axis=0) / gamma.sum(axis=0) + sensor = np.array([p_true, 1 - p_true]) + + return HiddenMarkovModel(A.tolist(), sensor.tolist(), prior.tolist()) + + +# _________________________________________________________________________ + + +def fixed_lag_smoothing(e_t, HMM, d, ev, t): + """ + [Figure 15.6] + Smoothing algorithm with a fixed time lag of 'd' steps. + Computes the smoothed estimate P(X_{t-d} | e_{1:t}) for the slice that lies + 'd' steps in the past, given the evidence sequence ev = [e_1, ..., e_t]. + Returns None when there is not yet enough evidence (t <= d). + """ + if t <= d: + return None + + T_model = np.array(HMM.transition_model) + + # forward message advanced over e_1 .. e_{t-d} + f = HMM.prior + for i in range(t - d): + f = forward(HMM, f, ev[i]) + + # backward transformation accumulated over the lag window e_{t-d+1} .. e_t + B = np.eye(len(f)) + for i in range(t - d, t): + O_i = np.diag(HMM.sensor_dist(ev[i])) + B = B @ T_model @ O_i + + return normalize((np.array(f) * (B @ np.ones(len(f)))).tolist()) + + +# _________________________________________________________________________ + + +def particle_filtering(e, N, HMM): + """Particle filtering considering two states variables.""" + dist = [0.5, 0.5] + # Weight Initialization + w = [0 for _ in range(N)] + # STEP 1 + # Propagate one step using transition model given prior state + dist = vector_add(scalar_vector_product(dist[0], HMM.transition_model[0]), + scalar_vector_product(dist[1], HMM.transition_model[1])) + # Assign state according to probability + s = ['A' if probability(dist[0]) else 'B' for _ in range(N)] + w_tot = 0 + # Calculate importance weight given evidence e + for i in range(N): + if s[i] == 'A': + # P(U|A)*P(A) + w_i = HMM.sensor_dist(e)[0] * dist[0] + if s[i] == 'B': + # P(U|B)*P(B) + w_i = HMM.sensor_dist(e)[1] * dist[1] + w[i] = w_i + w_tot += w_i + + # Normalize all the weights + for i in range(N): + w[i] = w[i] / w_tot + + # Limit weights to 4 digits + for i in range(N): + w[i] = float("{0:.4f}".format(w[i])) + + # STEP 2 + s = weighted_sample_with_replacement(N, s, w) + + return s + + +# _________________________________________________________________________ + + +class KalmanFilter: + """ + [Section 15.4] + Kalman filter for a linear-Gaussian dynamical system. The hidden state + evolves and is observed according to the linear-Gaussian model + + x_{t+1} = F x_t + noise, noise ~ N(0, Sigma_x) (transition model) + z_t = H x_t + noise, noise ~ N(0, Sigma_z) (sensor model) + + where F is the transition matrix, H the sensor matrix, Sigma_x the + transition (process) noise covariance and Sigma_z the sensor (measurement) + noise covariance. Because the family of Gaussians is closed under the + Bayesian filtering update, the forward message stays Gaussian and is fully + described by a mean vector and a covariance matrix at every step. + """ + + def __init__(self, transition_model, sensor_model, transition_noise, sensor_noise): + self.F = np.atleast_2d(transition_model) # transition matrix + self.H = np.atleast_2d(sensor_model) # sensor matrix + self.Sigma_x = np.atleast_2d(transition_noise) # transition noise covariance + self.Sigma_z = np.atleast_2d(sensor_noise) # sensor noise covariance + + def predict(self, mean, cov): + """Time update: project the Gaussian estimate one step forward through F.""" + mean = self.F @ mean + cov = self.F @ cov @ self.F.T + self.Sigma_x + return mean, cov + + def update(self, mean, cov, z): + """Measurement update: condition the predicted Gaussian on observation z.""" + # Kalman gain [Equation 15.21] + K = cov @ self.H.T @ np.linalg.inv(self.H @ cov @ self.H.T + self.Sigma_z) + mean = mean + K @ (np.atleast_1d(z) - self.H @ mean) + cov = (np.eye(cov.shape[0]) - K @ self.H) @ cov + return mean, cov + + def filter(self, mean, cov, z): + """One predict-then-update cycle for a single new observation z.""" + mean, cov = self.predict(mean, cov) + return self.update(mean, cov, z) + + +def kalman_filter(KF, mean0, cov0, observations): + """ + [Section 15.4] + Run the Kalman filter 'KF' over a sequence of 'observations', starting from + the Gaussian prior N(mean0, cov0). Returns, for each time step, the filtered + Gaussian estimate as a (mean, covariance) pair. + """ + mean, cov = np.atleast_1d(mean0).astype(float), np.atleast_2d(cov0).astype(float) + estimates = [] + for z in observations: + mean, cov = KF.filter(mean, cov, z) + estimates.append((mean, cov)) + + return estimates + + +# _________________________________________________________________________ + + +class DynamicBayesNet: + """ + [Section 15.5] + A dynamic Bayesian network for a stationary first-order Markov process. It is + specified by a prior network over the state variables at slice 0 and a single + transition + sensor network describing, for one time step, the distribution of + each state variable (given the previous slice) and of each evidence variable + (given the current slice). The DBN can be 'unrolled' into an ordinary BayesNet + spanning any number of slices and then queried with the exact inference + algorithms; in particular filtering is the query for the last state variable + given the whole evidence sequence. + + Each spec is a (variable, parents, cpt) triple as for a BayesNode. In a + transition spec, a parent named '_prev' refers to state variable at + the previous slice; every other parent refers to the current slice. + """ + + def __init__(self, prior, transition, sensors): + self.prior = prior + self.transition = transition + self.sensors = sensors + self.state_variables = [spec[0] for spec in prior] + self.evidence_variables = [spec[0] for spec in sensors] + + @staticmethod + def _rename(parents, t, t_prev): + """Map the parent names of a slice template to concrete unrolled names.""" + if isinstance(parents, str): + parents = parents.split() + return [f'{p[:-len("_prev")]}_{t_prev}' if p.endswith('_prev') else f'{p}_{t}' for p in parents] + + def unroll(self, steps): + """Unroll the DBN into a BayesNet over slices 0..steps (evidence at 1..steps).""" + specs = [(f'{var}_0', self._rename(parents, 0, 0), cpt) for var, parents, cpt in self.prior] + for t in range(1, steps + 1): + for var, parents, cpt in self.transition + self.sensors: + specs.append((f'{var}_{t}', self._rename(parents, t, t - 1), cpt)) + return BayesNet(specs) + + def filter(self, evidence, query, infer=elimination_ask): + """ + Filtering: the posterior over 'query' at the last slice given the whole + observation sequence. 'evidence' is a list of dicts, one per time step + t = 1, 2, ..., each mapping evidence variables to their observed values. + """ + steps = len(evidence) + net = self.unroll(steps) + e = {f'{var}_{t}': val for t, obs in enumerate(evidence, 1) for var, val in obs.items()} + return infer(f'{query}_{steps}', e, net) + + +# _________________________________________________________________________ + + +class MCLmap: + """Map which provides probability distributions and sensor readings. + Consists of discrete cells which are either an obstacle or empty""" + + def __init__(self, m): + self.m = m + self.nrows = len(m) + self.ncols = len(m[0]) + # list of empty spaces in the map + self.empty = [(i, j) for i in range(self.nrows) for j in range(self.ncols) if not m[i][j]] + + def sample(self): + """Returns a random kinematic state possible in the map""" + pos = random.choice(self.empty) + # 0N 1E 2S 3W + orient = random.choice(range(4)) + kin_state = pos + (orient,) + return kin_state + + def ray_cast(self, sensor_num, kin_state): + """Returns distance to nearest obstacle or map boundary in the direction of sensor""" + pos = kin_state[:2] + orient = kin_state[2] + # sensor layout when orientation is 0 (towards North) + # 0 + # 3R1 + # 2 + delta = ((sensor_num % 2 == 0) * (sensor_num - 1), (sensor_num % 2 == 1) * (2 - sensor_num)) + # sensor direction changes based on orientation + for _ in range(orient): + delta = (delta[1], -delta[0]) + range_count = 0 + while 0 <= pos[0] < self.nrows and 0 <= pos[1] < self.nrows and not self.m[pos[0]][pos[1]]: + pos = vector_add(pos, delta) + range_count += 1 + return range_count + + +def monte_carlo_localization(a, z, N, P_motion_sample, P_sensor, m, S=None): + """ + [Figure 25.9] + Monte Carlo localization algorithm + """ + + def ray_cast(sensor_num, kin_state, m): + return m.ray_cast(sensor_num, kin_state) + + M = len(z) + S_ = [0] * N + W_ = [0] * N + v = a['v'] + w = a['w'] + + if S is None: + S = [m.sample() for _ in range(N)] + + for i in range(N): + S_[i] = P_motion_sample(S[i], v, w) + W_[i] = 1 + for j in range(M): + z_ = ray_cast(j, S_[i], m) + W_[i] = W_[i] * P_sensor(z[j], z_) + + S = weighted_sample_with_replacement(N, S_, W_) + return S + + +class ContinuousMCLmap: + """[Figure 25.10] + A continuous 2-D map for Monte Carlo localization. The world is a rectangular + arena [0, width] x [0, height] containing axis-aligned rectangular obstacles. + A kinematic state is (x, y, heading) with x, y real-valued coordinates and + heading an angle in radians, so the robot is no longer snapped to a grid. + Range sensors cast rays from the robot to the nearest wall, returning + continuous distances rather than the integer step counts of the discrete + MCLmap. It exposes the same sample()/ray_cast(sensor_num, kin_state) interface + as MCLmap, so it plugs straight into monte_carlo_localization.""" + + def __init__(self, width, height, obstacles=(), + sensors=(0, math.pi / 2, math.pi, 3 * math.pi / 2)): + self.width = width + self.height = height + # each obstacle is a rectangle given as (x0, y0, x1, y1) with x0 <= x1, y0 <= y1 + self.obstacles = [tuple(o) for o in obstacles] + # sensor beam angles (radians) measured relative to the robot's heading + self.sensors = list(sensors) + self.segments = self._wall_segments() + + def _wall_segments(self): + """The arena boundary plus the four edges of every obstacle, each as a + ((x0, y0), (x1, y1)) line segment.""" + w, h = self.width, self.height + segments = [((0, 0), (w, 0)), ((w, 0), (w, h)), + ((w, h), (0, h)), ((0, h), (0, 0))] + for x0, y0, x1, y1 in self.obstacles: + segments += [((x0, y0), (x1, y0)), ((x1, y0), (x1, y1)), + ((x1, y1), (x0, y1)), ((x0, y1), (x0, y0))] + return segments + + def in_free_space(self, x, y): + """True if (x, y) lies inside the arena and outside every obstacle.""" + if not (0 <= x <= self.width and 0 <= y <= self.height): + return False + return not any(x0 <= x <= x1 and y0 <= y <= y1 for x0, y0, x1, y1 in self.obstacles) + + def sample(self): + """Return a random kinematic state (x, y, heading) in free space.""" + while True: + x = random.uniform(0, self.width) + y = random.uniform(0, self.height) + if self.in_free_space(x, y): + return x, y, random.uniform(0, 2 * math.pi) + + def ray_cast(self, sensor_num, kin_state): + """Distance from the robot to the nearest wall along the given sensor. + Casts the ray (x, y) + t * (cos a, sin a), with a = heading + sensor + offset, and returns the smallest positive intersection with any wall + segment (infinity if the ray escapes the arena, which cannot happen for + a robot inside a closed arena).""" + x, y, heading = kin_state + angle = heading + self.sensors[sensor_num] + dx, dy = math.cos(angle), math.sin(angle) + nearest = math.inf + for (ax, ay), (bx, by) in self.segments: + ex, ey = bx - ax, by - ay + # ray-segment intersection: dot of segment with the ray normal (-dy, dx) + denom = ex * (-dy) + ey * dx + if abs(denom) < 1e-12: + continue # ray parallel to this wall segment + t = (ex * (y - ay) - ey * (x - ax)) / denom # distance along the ray + u = ((x - ax) * (-dy) + (y - ay) * dx) / denom # position along the segment + if t >= 0 and 0 <= u <= 1: + nearest = min(nearest, t) + return nearest diff --git a/q-learning-9b/aima/reinforcement_learning.py b/q-learning-9b/aima/reinforcement_learning.py new file mode 100644 index 0000000..60c3a38 --- /dev/null +++ b/q-learning-9b/aima/reinforcement_learning.py @@ -0,0 +1,376 @@ +"""Reinforcement Learning (Chapter 21)""" + +import random +from collections import defaultdict + +from aima.mdp import MDP, policy_evaluation + + +class PassiveDUEAgent: + """ + Passive (non-learning) agent that uses direct utility estimation + on a given MDP and policy:: + + import sys + from aima.mdp import sequential_decision_environment + north = (0, 1) + south = (0,-1) + west = (-1, 0) + east = (1, 0) + policy = {(0, 2): east, (1, 2): east, (2, 2): east, (3, 2): None, (0, 1): north, (2, 1): north, + (3, 1): None, (0, 0): north, (1, 0): west, (2, 0): west, (3, 0): west,} + agent = PassiveDUEAgent(policy, sequential_decision_environment) + for i in range(200): + run_single_trial(agent,sequential_decision_environment) + agent.estimate_U() + agent.U[(0, 0)] > 0.2 + True + """ + + def __init__(self, pi, mdp): + self.pi = pi + self.mdp = mdp + self.U = {} + self.s = None + self.a = None + self.s_history = [] + self.r_history = [] + self.init = mdp.init + + def __call__(self, percept): + s1, r1 = percept + self.s_history.append(s1) + self.r_history.append(r1) + ## + ## + if s1 in self.mdp.terminals: + self.s = self.a = None + else: + self.s, self.a = s1, self.pi[s1] + return self.a + + def estimate_U(self): + """Update utility estimates from the most recent completed trial by direct + utility estimation: average the observed reward-to-go for each visited state, + blend it with the running estimate, and reset the trial history. Must be + called only once the MDP has reached a terminal state.""" + # this function can be called only if the MDP has reached a terminal state + # it will also reset the mdp history + assert self.a is None, 'MDP is not in terminal state' + assert len(self.s_history) == len(self.r_history) + # calculating the utilities based on the current iteration + U2 = {s: [] for s in set(self.s_history)} + for i in range(len(self.s_history)): + s = self.s_history[i] + U2[s] += [sum(self.r_history[i:])] + U2 = {k: sum(v) / max(len(v), 1) for k, v in U2.items()} + # resetting history + self.s_history, self.r_history = [], [] + # setting the new utilities to the average of the previous + # iteration and this one + for k in U2.keys(): + if k in self.U.keys(): + self.U[k] = (self.U[k] + U2[k]) / 2 + else: + self.U[k] = U2[k] + return self.U + + def update_state(self, percept): + """To be overridden in most cases. The default case + assumes the percept to be of type (state, reward)""" + return percept + + +class PassiveADPAgent: + """ + [Figure 21.2] + Passive (non-learning) agent that uses adaptive dynamic programming + on a given MDP and policy:: + + import sys + from aima.mdp import sequential_decision_environment + north = (0, 1) + south = (0,-1) + west = (-1, 0) + east = (1, 0) + policy = {(0, 2): east, (1, 2): east, (2, 2): east, (3, 2): None, (0, 1): north, (2, 1): north, + (3, 1): None, (0, 0): north, (1, 0): west, (2, 0): west, (3, 0): west,} + agent = PassiveADPAgent(policy, sequential_decision_environment) + for i in range(100): + run_single_trial(agent,sequential_decision_environment) + + agent.U[(0, 0)] > 0.2 + True + agent.U[(0, 1)] > 0.2 + True + """ + + class ModelMDP(MDP): + """Class for implementing modified Version of input MDP with + an editable transition model P and a custom function T.""" + + def __init__(self, init, actlist, terminals, gamma, states): + super().__init__(init, actlist, terminals, states=states, gamma=gamma) + nested_dict = lambda: defaultdict(nested_dict) + # StackOverflow:whats-the-best-way-to-initialize-a-dict-of-dicts-in-python + self.P = nested_dict() + + def T(self, s, a): + """Return a list of tuples with probabilities for states + based on the learnt model P.""" + return [(prob, res) for (res, prob) in self.P[(s, a)].items()] + + def __init__(self, pi, mdp): + self.pi = pi + self.mdp = PassiveADPAgent.ModelMDP(mdp.init, mdp.actlist, + mdp.terminals, mdp.gamma, mdp.states) + self.U = {} + self.Nsa = defaultdict(int) + self.Ns1_sa = defaultdict(int) + self.s = None + self.a = None + self.visited = set() # keeping track of visited states + + def __call__(self, percept): + s1, r1 = percept + mdp = self.mdp + R, P, terminals, pi = mdp.reward, mdp.P, mdp.terminals, self.pi + s, a, Nsa, Ns1_sa, U = self.s, self.a, self.Nsa, self.Ns1_sa, self.U + + if s1 not in self.visited: # Reward is only known for visited state. + U[s1] = R[s1] = r1 + self.visited.add(s1) + if s is not None: + Nsa[(s, a)] += 1 + Ns1_sa[(s1, s, a)] += 1 + # for each t such that Ns′|sa [t, s, a] is nonzero + for t in [res for (res, state, act), freq in Ns1_sa.items() + if (state, act) == (s, a) and freq != 0]: + P[(s, a)][t] = Ns1_sa[(t, s, a)] / Nsa[(s, a)] + + self.U = policy_evaluation(pi, U, mdp) + ## + ## + self.Nsa, self.Ns1_sa = Nsa, Ns1_sa + if s1 in terminals: + self.s = self.a = None + else: + self.s, self.a = s1, self.pi[s1] + return self.a + + def update_state(self, percept): + """To be overridden in most cases. The default case + assumes the percept to be of type (state, reward).""" + return percept + + +class PassiveTDAgent: + """ + [Figure 21.4] + The abstract class for a Passive (non-learning) agent that uses + temporal differences to learn utility estimates. Override update_state + method to convert percept to state and reward. The mdp being provided + should be an instance of a subclass of the MDP Class:: + + import sys + from aima.mdp import sequential_decision_environment + north = (0, 1) + south = (0,-1) + west = (-1, 0) + east = (1, 0) + policy = {(0, 2): east, (1, 2): east, (2, 2): east, (3, 2): None, (0, 1): north, (2, 1): north, + (3, 1): None, (0, 0): north, (1, 0): west, (2, 0): west, (3, 0): west,} + agent = PassiveTDAgent(policy, sequential_decision_environment, alpha=lambda n: 60./(59+n)) + for i in range(200): + run_single_trial(agent,sequential_decision_environment) + + agent.U[(0, 0)] > 0.2 + True + agent.U[(0, 1)] > 0.2 + True + """ + + def __init__(self, pi, mdp, alpha=None): + + self.pi = pi + self.U = {s: 0. for s in mdp.states} + self.Ns = {s: 0 for s in mdp.states} + self.s = None + self.a = None + self.r = None + self.gamma = mdp.gamma + self.terminals = mdp.terminals + + if alpha: + self.alpha = alpha + else: + self.alpha = lambda n: 1 / (1 + n) # udacity video + + def __call__(self, percept): + s1, r1 = self.update_state(percept) + pi, U, Ns, s, r = self.pi, self.U, self.Ns, self.s, self.r + alpha, gamma, terminals = self.alpha, self.gamma, self.terminals + if not Ns[s1]: + U[s1] = r1 + if s is not None: + Ns[s] += 1 + U[s] += alpha(Ns[s]) * (r + gamma * U[s1] - U[s]) + if s1 in terminals: + self.s = self.a = self.r = None + else: + self.s, self.a, self.r = s1, pi[s1], r1 + return self.a + + def update_state(self, percept): + """To be overridden in most cases. The default case + assumes the percept to be of type (state, reward).""" + return percept + + +class QLearningAgent: + """ + [Figure 21.8] + An exploratory Q-learning agent. It avoids having to learn the transition + model because the Q-value of a state can be related directly to those of + its neighbors:: + + import sys + from aima.mdp import sequential_decision_environment + north = (0, 1) + south = (0,-1) + west = (-1, 0) + east = (1, 0) + policy = {(0, 2): east, (1, 2): east, (2, 2): east, (3, 2): None, (0, 1): north, (2, 1): north, + (3, 1): None, (0, 0): north, (1, 0): west, (2, 0): west, (3, 0): west,} + q_agent = QLearningAgent(sequential_decision_environment, Ne=5, Rplus=2, alpha=lambda n: 60./(59+n)) + for i in range(200): + run_single_trial(q_agent,sequential_decision_environment) + + q_agent.Q[((0, 1), (0, 1))] >= -0.5 + True + q_agent.Q[((1, 0), (0, -1))] <= 0.5 + True + """ + + def __init__(self, mdp, Ne, Rplus, alpha=None): + + self.gamma = mdp.gamma + self.terminals = mdp.terminals + self.all_act = mdp.actlist + self.Ne = Ne # iteration limit in exploration function + self.Rplus = Rplus # large value to assign before iteration limit + self.Q = defaultdict(float) + self.Nsa = defaultdict(float) + self.s = None + self.a = None + self.r = None + + if alpha: + self.alpha = alpha + else: + self.alpha = lambda n: 1. / (1 + n) # udacity video + + def f(self, u, n): + """Exploration function. Returns fixed Rplus until + agent has visited state, action a Ne number of times. + Same as ADP agent in book.""" + if n < self.Ne: + return self.Rplus + else: + return u + + def actions_in_state(self, state): + """Return actions possible in given state. + Useful for max and argmax.""" + if state in self.terminals: + return [None] + else: + return self.all_act + + def __call__(self, percept): + s1, r1 = self.update_state(percept) + Q, Nsa, s, a, r = self.Q, self.Nsa, self.s, self.a, self.r + alpha, gamma, terminals = self.alpha, self.gamma, self.terminals, + actions_in_state = self.actions_in_state + + if s in terminals: + Q[s, None] = r + if s is not None: + Nsa[s, a] += 1 + Q[s, a] += alpha(Nsa[s, a]) * (r + gamma * max(Q[s1, a1] + for a1 in actions_in_state(s1)) - Q[s, a]) + if s in terminals: + self.s = self.a = self.r = None + else: + self.s, self.r = s1, r1 + self.a = max(actions_in_state(s1), key=lambda a1: self.f(Q[s1, a1], Nsa[s1, a1])) + return self.a + + def update_state(self, percept): + """To be overridden in most cases. The default case + assumes the percept to be of type (state, reward).""" + return percept + + +class SARSALearningAgent(QLearningAgent): + """ + [Section 21.3] + An on-policy temporal-difference control agent (SARSA: State-Action-Reward- + State-Action). It is identical to the Q-learning agent except for the update + rule: instead of bootstrapping on the maximum Q-value over next actions, SARSA + bootstraps on the Q-value of the action a1 that its exploration policy will + actually take in the next state. Being on-policy, SARSA learns the value of the + policy it is following, exploration included, rather than that of the greedy + policy. + """ + + def __call__(self, percept): + s1, r1 = self.update_state(percept) + Q, Nsa, s, a, r = self.Q, self.Nsa, self.s, self.a, self.r + alpha, gamma, terminals = self.alpha, self.gamma, self.terminals + actions_in_state = self.actions_in_state + + # pick the next action with the same exploration policy as Q-learning + a1 = max(actions_in_state(s1), key=lambda a2: self.f(Q[s1, a2], Nsa[s1, a2])) + + if s in terminals: + Q[s, None] = r + if s is not None: + Nsa[s, a] += 1 + # on-policy update: bootstrap on the actually-chosen next action a1 + Q[s, a] += alpha(Nsa[s, a]) * (r + gamma * Q[s1, a1] - Q[s, a]) + if s in terminals: + self.s = self.a = self.r = None + else: + self.s, self.r = s1, r1 + self.a = a1 + return self.a + + +def run_single_trial(agent_program, mdp): + """Execute trial for given agent_program + and mdp. mdp should be an instance of subclass + of mdp.MDP """ + + def take_single_action(mdp, s, a): + """ + Select outcome of taking action a + in state s. Weighted Sampling. + """ + x = random.uniform(0, 1) + cumulative_probability = 0.0 + for probability_state in mdp.T(s, a): + probability, state = probability_state + cumulative_probability += probability + if x < cumulative_probability: + break + return state + + current_state = mdp.init + while True: + current_reward = mdp.R(current_state) + percept = (current_state, current_reward) + next_action = agent_program(percept) + if next_action is None: + break + current_state = take_single_action(mdp, current_state, next_action) diff --git a/q-learning-9b/aima/search.py b/q-learning-9b/aima/search.py new file mode 100644 index 0000000..f379f6d --- /dev/null +++ b/q-learning-9b/aima/search.py @@ -0,0 +1,2067 @@ +""" +Search (Chapters 3-4) + +The way to use this code is to subclass Problem to create a class of problems, +then create problem instances and solve them with calls to the various search +functions. +""" + +import sys +from collections import deque + +from aima.utils import * + + +class Problem: + """The abstract class for a formal problem. You should subclass + this and implement the methods actions and result, and possibly + __init__, goal_test, and path_cost. Then you will create instances + of your subclass and solve them with the various search functions.""" + + def __init__(self, initial, goal=None): + """The constructor specifies the initial state, and possibly a goal + state, if there is a unique goal. Your subclass's constructor can add + other arguments.""" + self.initial = initial + self.goal = goal + + def actions(self, state): + """Return the actions that can be executed in the given + state. The result would typically be a list, but if there are + many actions, consider yielding them one at a time in an + iterator, rather than building them all at once.""" + raise NotImplementedError + + def result(self, state, action): + """Return the state that results from executing the given + action in the given state. The action must be one of + self.actions(state).""" + raise NotImplementedError + + def goal_test(self, state): + """Return True if the state is a goal. The default method compares the + state to self.goal or checks for state in self.goal if it is a + list, as specified in the constructor. Override this method if + checking against a single self.goal is not enough.""" + if isinstance(self.goal, list): + return is_in(state, self.goal) + else: + return state == self.goal + + def path_cost(self, c, state1, action, state2): + """Return the cost of a solution path that arrives at state2 from + state1 via action, assuming cost c to get up to state1. If the problem + is such that the path doesn't matter, this function will only look at + state2. If the path does matter, it will consider c and maybe state1 + and action. The default method costs 1 for every step in the path.""" + return c + 1 + + def value(self, state): + """For optimization problems, each state has a value. Hill Climbing + and related algorithms try to maximize this value.""" + raise NotImplementedError + + +# ______________________________________________________________________________ + + +class Node: + """A node in a search tree. Contains a pointer to the parent (the node + that this is a successor of) and to the actual state for this node. Note + that if a state is arrived at by two paths, then there are two nodes with + the same state. Also includes the action that got us to this state, and + the total path_cost (also known as g) to reach the node. Other functions + may add an f and h value; see best_first_graph_search and astar_search for + an explanation of how the f and h values are handled. You will not need to + subclass this class.""" + + def __init__(self, state, parent=None, action=None, path_cost=0): + """Create a search tree Node, derived from a parent by an action.""" + self.state = state + self.parent = parent + self.action = action + self.path_cost = path_cost + self.depth = 0 + if parent: + self.depth = parent.depth + 1 + + def __repr__(self): + return "".format(self.state) + + def __lt__(self, node): + return self.state < node.state + + def expand(self, problem): + """List the nodes reachable in one step from this node.""" + return [self.child_node(problem, action) + for action in problem.actions(self.state)] + + def child_node(self, problem, action): + """[Figure 3.10]""" + next_state = problem.result(self.state, action) + next_node = Node(next_state, self, action, problem.path_cost(self.path_cost, self.state, action, next_state)) + return next_node + + def solution(self): + """Return the sequence of actions to go from the root to this node.""" + return [node.action for node in self.path()[1:]] + + def path(self): + """Return a list of nodes forming the path from the root to this node.""" + node, path_back = self, [] + while node: + path_back.append(node) + node = node.parent + return list(reversed(path_back)) + + def path_states(self): + """Return the sequence of states on the path from the root to this node. + + The search functions return the goal ``Node`` (staying close to the book's + pseudocode); this is the one-call way to read off the whole solution path as + states -- ``solution()`` gives the corresponding actions. + """ + return [node.state for node in self.path()] + + # We want for a queue of nodes in breadth_first_graph_search or + # astar_search to have no duplicated states, so we treat nodes + # with the same state as equal. [Problem: this may not be what you + # want in other contexts.] + + def __eq__(self, other): + return isinstance(other, Node) and self.state == other.state + + def __hash__(self): + # We use the hash value of the state + # stored in the node instead of the node + # object itself to quickly search a node + # with the same state in a Hash Table + return hash(self.state) + + +# ______________________________________________________________________________ + + +class SimpleProblemSolvingAgentProgram: + """ + [Figure 3.1] + Abstract framework for a problem-solving agent. + """ + + def __init__(self, initial_state=None): + """State is an abstract representation of the state + of the world, and seq is the list of actions required + to get to a particular state from the initial state(root).""" + self.state = initial_state + self.seq = [] + + def __call__(self, percept): + """[Figure 3.1] Formulate a goal and problem, then + search for a sequence of actions to solve it.""" + self.state = self.update_state(self.state, percept) + if not self.seq: + goal = self.formulate_goal(self.state) + problem = self.formulate_problem(self.state, goal) + self.seq = self.search(problem) + if not self.seq: + return None + return self.seq.pop(0) + + def update_state(self, state, percept): + """Update the agent's world state from the latest percept (abstract).""" + raise NotImplementedError + + def formulate_goal(self, state): + """Decide on a goal to pursue given the current state (abstract).""" + raise NotImplementedError + + def formulate_problem(self, state, goal): + """Build a Problem instance from the current state and chosen goal (abstract).""" + raise NotImplementedError + + def search(self, problem): + """Return a sequence of actions that solves the given problem (abstract).""" + raise NotImplementedError + + +# ______________________________________________________________________________ +# Uninformed Search algorithms + + +def breadth_first_tree_search(problem): + """ + [Figure 3.7] + Search the shallowest nodes in the search tree first. + Search through the successors of a problem to find a goal. + The argument frontier should be an empty queue. + Repeats infinitely in case of loops. + """ + + frontier = deque([Node(problem.initial)]) # FIFO queue + + while frontier: + node = frontier.popleft() + if problem.goal_test(node.state): + return node + frontier.extend(node.expand(problem)) + return None + + +def depth_first_tree_search(problem): + """ + [Figure 3.7] + Search the deepest nodes in the search tree first. + Search through the successors of a problem to find a goal. + The argument frontier should be an empty queue. + Repeats infinitely in case of loops. + """ + + frontier = [Node(problem.initial)] # Stack + + while frontier: + node = frontier.pop() + if problem.goal_test(node.state): + return node + frontier.extend(node.expand(problem)) + return None + + +def depth_first_graph_search(problem): + """ + [Figure 3.7] + Search the deepest nodes in the search tree first. + Search through the successors of a problem to find a goal. + The argument frontier should be an empty queue. + Does not get trapped by loops. + If two paths reach a state, only use the first one. + """ + frontier = [(Node(problem.initial))] # Stack + + explored = set() + while frontier: + node = frontier.pop() + if problem.goal_test(node.state): + return node + explored.add(node.state) + frontier.extend(child for child in node.expand(problem) + if child.state not in explored and child not in frontier) + return None + + +def breadth_first_graph_search(problem): + """[Figure 3.11] + Note that this function can be implemented in a + single line as below: + return graph_search(problem, FIFOQueue()) + """ + node = Node(problem.initial) + if problem.goal_test(node.state): + return node + frontier = deque([node]) + explored = set() + while frontier: + node = frontier.popleft() + explored.add(node.state) + for child in node.expand(problem): + if child.state not in explored and child not in frontier: + if problem.goal_test(child.state): + return child + frontier.append(child) + return None + + +def best_first_graph_search(problem, f, display=False): + """Search the nodes with the lowest f scores first. + You specify the function f(node) that you want to minimize; for example, + if f is a heuristic estimate to the goal, then we have greedy best + first search; if f is node.depth then we have breadth-first search. + There is a subtlety: the line "f = memoize(f, 'f')" means that the f + values will be cached on the nodes as they are computed. So after doing + a best first search you can examine the f values of the path returned.""" + f = memoize(f, 'f') + node = Node(problem.initial) + frontier = PriorityQueue('min', f) + frontier.append(node) + explored = set() + while frontier: + node = frontier.pop() + if problem.goal_test(node.state): + if display: + print(len(explored), "paths have been expanded and", len(frontier), "paths remain in the frontier") + return node + explored.add(node.state) + for child in node.expand(problem): + if child.state not in explored and child not in frontier: + frontier.append(child) + elif child in frontier: + if f(child) < frontier[child]: + del frontier[child] + frontier.append(child) + return None + + +def best_first_tree_search(problem, f, display=False): + """Search the nodes with the lowest f scores first, without remembering the + states already reached. This is the tree-search counterpart of + best_first_graph_search: it drops the explored set and the frontier + membership/replacement bookkeeping, so it is faster when the state space is a + tree (no repeated states) but does not terminate on a graph that contains + cycles. You specify the function f(node) that you want to minimize.""" + f = memoize(f, 'f') + node = Node(problem.initial) + frontier = PriorityQueue('min', f) + frontier.append(node) + while frontier: + node = frontier.pop() + if problem.goal_test(node.state): + if display: + print(len(frontier), "paths remain in the frontier") + return node + for child in node.expand(problem): + frontier.append(child) + return None + + +def uniform_cost_search(problem, display=False): + """[Figure 3.14]""" + return best_first_graph_search(problem, lambda node: node.path_cost, display) + + +def uniform_cost_tree_search(problem, display=False): + """[Figure 3.14] Tree-search version of uniform-cost search.""" + return best_first_tree_search(problem, lambda node: node.path_cost, display) + + +def depth_limited_search(problem, limit=50): + """[Figure 3.17]""" + + def recursive_dls(node, problem, limit): + if problem.goal_test(node.state): + return node + elif limit == 0: + return 'cutoff' + else: + cutoff_occurred = False + for child in node.expand(problem): + result = recursive_dls(child, problem, limit - 1) + if result == 'cutoff': + cutoff_occurred = True + elif result is not None: + return result + return 'cutoff' if cutoff_occurred else None + + # Body of depth_limited_search: + return recursive_dls(Node(problem.initial), problem, limit) + + +def iterative_deepening_search(problem): + """[Figure 3.18]""" + for depth in range(sys.maxsize): + result = depth_limited_search(problem, depth) + if result != 'cutoff': + return result + + +# ______________________________________________________________________________ +# Bidirectional Search +# Pseudocode from https://webdocs.cs.ualberta.ca/%7Eholte/Publications/MM-AAAI2016.pdf + +def bidirectional_search(problem): + """Meet-in-the-middle (MM) bidirectional search. Searches forward from the + initial state and backward from the goal at the same time, returning the cost + of the optimal path where the two frontiers meet (np.inf if no path exists).""" + e = 0 + if isinstance(problem, GraphProblem): + e = problem.find_min_edge() + gF, gB = {Node(problem.initial): 0}, {Node(problem.goal): 0} + openF, openB = [Node(problem.initial)], [Node(problem.goal)] + closedF, closedB = [], [] + U = np.inf + + def extend(U, open_dir, open_other, g_dir, g_other, closed_dir): + """Extend search in given direction""" + n = find_key(C, open_dir, g_dir) + + open_dir.remove(n) + closed_dir.append(n) + + for c in n.expand(problem): + if c in open_dir or c in closed_dir: + if g_dir[c] <= problem.path_cost(g_dir[n], n.state, None, c.state): + continue + + open_dir.remove(c) + + g_dir[c] = problem.path_cost(g_dir[n], n.state, None, c.state) + open_dir.append(c) + + if c in open_other: + U = min(U, g_dir[c] + g_other[c]) + + return U, open_dir, closed_dir, g_dir + + def find_min(open_dir, g): + """Finds minimum priority, g and f values in open_dir""" + # pr_min_f isn't forward pr_min instead it's the f-value + # of node with priority pr_min. + pr_min, pr_min_f = np.inf, np.inf + for n in open_dir: + f = g[n] + problem.h(n) + pr = max(f, 2 * g[n]) + pr_min = min(pr_min, pr) + pr_min_f = min(pr_min_f, f) + + return pr_min, pr_min_f, min(g.values()) + + def find_key(pr_min, open_dir, g): + """Finds key in open_dir with value equal to pr_min + and minimum g value.""" + m = np.inf + node = Node(-1) + for n in open_dir: + pr = max(g[n] + problem.h(n), 2 * g[n]) + if pr == pr_min: + if g[n] < m: + m = g[n] + node = n + + return node + + while openF and openB: + pr_min_f, f_min_f, g_min_f = find_min(openF, gF) + pr_min_b, f_min_b, g_min_b = find_min(openB, gB) + C = min(pr_min_f, pr_min_b) + + if U <= max(C, f_min_f, f_min_b, g_min_f + g_min_b + e): + return U + + if C == pr_min_f: + # Extend forward + U, openF, closedF, gF = extend(U, openF, openB, gF, gB, closedF) + else: + # Extend backward + U, openB, closedB, gB = extend(U, openB, openF, gB, gF, closedB) + + return np.inf + + +# ______________________________________________________________________________ +# Informed (Heuristic) Search + + +greedy_best_first_graph_search = best_first_graph_search +greedy_best_first_tree_search = best_first_tree_search + + +# Greedy best-first search is accomplished by specifying f(n) = h(n). + + +def astar_search(problem, h=None, display=False): + """A* search is best-first graph search with f(n) = g(n)+h(n). + You need to specify the h function when you call astar_search, or + else in your Problem subclass.""" + h = memoize(h or problem.h, 'h') + return best_first_graph_search(problem, lambda n: n.path_cost + h(n), display) + + +def astar_tree_search(problem, h=None, display=False): + """Tree-search version of A*: best-first tree search with f(n) = g(n) + h(n). + Faster than astar_search when the state space is a tree, since it keeps no + explored set. You need to specify the h function when you call + astar_tree_search, or else in your Problem subclass.""" + h = memoize(h or problem.h, 'h') + return best_first_tree_search(problem, lambda n: n.path_cost + h(n), display) + + +def iterative_deepening_astar_search(problem, h=None): + """[Section 3.5.3] Iterative-deepening A* search: repeatedly run a depth-first + search bounded by an f = g + h contour, raising the bound to the smallest f + that exceeded it, until a goal within the bound is found.""" + h = memoize(h or problem.h, 'h') + + def f(node): + return node.path_cost + h(node) + + def contour(node, bound): + """Depth-first search pruned at f(node) > bound. Return a goal Node, or + the smallest f-value among the nodes that exceeded the bound.""" + if f(node) > bound: + return f(node) + if problem.goal_test(node.state): + return node + minimum = np.inf + for child in node.expand(problem): + # avoid cycles along the current path + if child.state not in (ancestor.state for ancestor in node.path()): + result = contour(child, bound) + if isinstance(result, Node): + return result + minimum = min(minimum, result) + return minimum + + node = Node(problem.initial) + bound = f(node) + while True: + result = contour(node, bound) + if isinstance(result, Node): + return result + if result == np.inf: + return None + bound = result + + +# ______________________________________________________________________________ +# A* heuristics + +class EightPuzzle(Problem): + """ The problem of sliding tiles numbered from 1 to 8 on a 3x3 board, where one of the + squares is a blank. A state is represented as a tuple of length 9, where element at + index i represents the tile number at index i (0 if it's an empty square) """ + + def __init__(self, initial, goal=(1, 2, 3, 4, 5, 6, 7, 8, 0)): + """ Define goal state and initialize a problem """ + super().__init__(initial, goal) + + def find_blank_square(self, state): + """Return the index of the blank square in a given state""" + + return state.index(0) + + def actions(self, state): + """ Return the actions that can be executed in the given state. + The result would be a list, since there are only four possible actions + in any given state of the environment """ + + possible_actions = ['UP', 'DOWN', 'LEFT', 'RIGHT'] + index_blank_square = self.find_blank_square(state) + + if index_blank_square % 3 == 0: + possible_actions.remove('LEFT') + if index_blank_square < 3: + possible_actions.remove('UP') + if index_blank_square % 3 == 2: + possible_actions.remove('RIGHT') + if index_blank_square > 5: + possible_actions.remove('DOWN') + + return possible_actions + + def result(self, state, action): + """ Given state and action, return a new state that is the result of the action. + Action is assumed to be a valid action in the state """ + + # blank is the index of the blank square + blank = self.find_blank_square(state) + new_state = list(state) + + delta = {'UP': -3, 'DOWN': 3, 'LEFT': -1, 'RIGHT': 1} + neighbor = blank + delta[action] + new_state[blank], new_state[neighbor] = new_state[neighbor], new_state[blank] + + return tuple(new_state) + + def goal_test(self, state): + """ Given a state, return True if state is a goal state or False, otherwise """ + + return state == self.goal + + def check_solvability(self, state): + """ Checks if the given state is solvable """ + + inversion = 0 + for i in range(len(state)): + for j in range(i + 1, len(state)): + if (state[i] > state[j]) and state[i] != 0 and state[j] != 0: + inversion += 1 + + return inversion % 2 == 0 + + def h(self, node): + """ Return the heuristic value for a given state. Default heuristic function used is + h(n) = number of misplaced tiles """ + + return sum(s != g for (s, g) in zip(node.state, self.goal)) + + +# ______________________________________________________________________________ + +class NPuzzle(Problem): + """Generalization of the Eight Puzzle problem. The problem consists of sliding tiles numbered from 1 to N ^2 on a NxN board, + where one of the squares is a blank. The state is represented as a tuple of length N^2, + where element at index i represents the tile number at index i , where '0' denotes empty square index""" + def __init__(self, initial=None, goal=None, size=3, heuristic='hamming', shuffle=10): + """Args: + intial: intiail state of puzzle. If not passed will default to goal state + size: number of rows and columns in puzzle + heuristic: heuristic used in astat search + shuffle: number of times to perfrom random action from the initial state. + Returns: + Problem instance modeling any size of sliding puzzle""" + self.size = size + self.heuristic = heuristic + goal = goal or (tuple(range(1,size*size, 1)) + (0, )) + initial = initial or (tuple(range(1,size*size, 1)) + (0, )) + if shuffle: + initial = self.shuffle_state(shuffle, list(initial)) + assert len(goal) == size*size, "length of goal tuple should be {length}".format(length=size*size) + assert len(initial) == size * size, "length of initial tuple should be {length}".format(length=size * size) + assert self.check_solvability(initial), "The puzzle is not solvable from the given initial state!" + Problem.__init__(self, initial, goal) + + def shuffle_state(self, shuffle, state): + """Perform random action from the initial state""" + for _ in range(shuffle): + action = random.sample(self.actions(state),1)[0] + state = self.result(state,action) + return state + + def find_blank_square(self, state): + """Return the index of the blank square in a given state""" + return state.index(0) + + def actions(self, state): + """Return the actions that can be executed in the given state. + The result would be a list of maximally four directions, since there are only four possible actions + in any given state of the environment""" + possible_actions = ['UP', 'DOWN', 'LEFT', 'RIGHT'] + index_blank_square = self.find_blank_square(state) + if index_blank_square % self.size == 0: + possible_actions.remove('LEFT') + if index_blank_square < self.size: + possible_actions.remove('UP') + if index_blank_square % self.size == self.size -1 : + possible_actions.remove('RIGHT') + if index_blank_square > self.size*self.size - self.size - 1: + possible_actions.remove('DOWN') + return possible_actions + + def result(self, state, action): + """Given state and action, return a new state that is the result of the action. + Action is assumed to be a valid action in the state""" + index_blank_square = self.find_blank_square(state) + new_state = list(state) + delta = {'UP': -self.size, 'DOWN': self.size, 'LEFT': -1, 'RIGHT': 1} + neighbor = index_blank_square + delta[action] + new_state[index_blank_square], new_state[neighbor] = new_state[neighbor], new_state[index_blank_square] + return tuple(new_state) + + def goal_test(self, state): + """Given a state, return True if state is a goal state or False otherwise""" + return state == self.goal + + def check_solvability(self, state): + """Checks if the given state is solvable. Whether or not puzzle is solvable depends on size being even or odd. + If N is odd, then puzzle instance is solvable if number of inversions is even in the input state. + If N is even, puzzle instance is solvable if the blank is on an even row counting from the bottom (second-last, fourth-last, etc.) and number of inversions is odd + or the blank is on an odd row counting from the bottom (last, third-last, fifth-last, etc.) and number of inversions is even." + reference : https://www.geeksforgeeks.org/check-instance-15-puzzle-solvable/""" + solvable = True + inversions = 0 + for i in range(len(state)-1): + for j in range(i+1, len(state)): + if state[i] != 0 and state[j] != 0 and state[i] > state[j]: + inversions += 1 + if self.size % 2 != 0: + solvable = True if (inversions % 2 == 0 ) else False + elif self.find_blank_square(state) // self.size % 2 == 0 and inversions % 2 != 0 : + solvable = True + elif self.find_blank_square(state) // self.size % 2 != 0 and inversions % 2 == 0: + solvable = True + else: + solvable = False + return solvable + + def h(self, node): + """Return the heuristic value for a given state. Default heuristic function used is + h(n) = number of misplaced tiles""" + heuristics = {'hamming':self.hamming_distance_heuristic(node)} + return heuristics.get(self.heuristic) + + def hamming_distance_heuristic(self, node): + """Number of tiles that are not in their goal position (Hamming distance).""" + return hamming_distance(node.state, self.goal) + + +class TravelingSalesman(Problem): + """ The problem of finding the shortest Hamiltonian cycle (tour) that visits + every city exactly once and returns to the start.""" + + def __init__(self, all_cities, initial, goal=None): + # initial = (0), goal = some state that its first and last values are 0 + """ Define goal state and initialize a problem """ + super().__init__(initial, goal) + self.cities_matrix = self.init_distance_matrix(all_cities) + self.cities_identifiers = set(all_cities.keys()) + self.cities_amount = len(self.cities_identifiers) + self.trees_evaluation_hash = {} + + def value(self, state): + """ Define state value """ + overall_distance = 0.0 + state_max_ind = len(state) - 1 + for ind, city in enumerate(state): + if ind != state_max_ind: + overall_distance += self.cities_matrix[state[ind]][state[ind + 1]] + return overall_distance + + def actions(self, state): + """ Return the actions that can be executed in the given state. + The result would be a list""" + if len(state) == self.cities_amount: + return [0] + return self.get_unvisited_cities(state) + + def result(self, state, action): + """ Given state and action, return a new state that is the result of the action. + Action is assumed to be a valid action in the state """ + + # blank is the index of the blank square + new_state = list(state) + new_state.append(action) + return tuple(new_state) + + def goal_test(self, state): + """ Given a state, return True if state is a goal state or False, otherwise""" + return len(state) == (self.cities_amount + 1) and state[0] == state[self.cities_amount] + + def get_unvisited_cities(self, state): + """ Returns unused actions(cities) """ + return self.cities_identifiers.difference(state) + + def path_cost(self, c, state1, action, state2): + """ Return next state cost """ + return c + self.cities_matrix[state1[-1]][state2[-1]] + + def h(self, node): + """Admissible heuristic: total edge weight of the MST over the cities + that have not yet been visited.""" + h = 0.0 + unvisited_cities = self.get_unvisited_cities(node.state) + mst_evaluation = self.eval_unvisited_mst_edges_sum(unvisited_cities) + h += mst_evaluation + return h + + def eval_unvisited_mst_edges_sum(self, unvisited_nodes): + """Return the total MST edge weight over the unvisited cities (0.0 if none).""" + if not unvisited_nodes: + return 0.0 + return self.tsp_kruskal(unvisited_nodes) + + def tsp_kruskal(self, unvisited_nodes): + """ Generate MST of the graph and return sum of edges """ + forest_value = 0.0 + disjoint_sets = DisjointSets() + non_sorted_edges_list = [] + for vertex_x in unvisited_nodes: + disjoint_sets.make_set(vertex_x) + for vertex_y in unvisited_nodes: + if vertex_x < vertex_y: + non_sorted_edges_list.append((vertex_x, vertex_y)) + edges_list = sorted(non_sorted_edges_list, key=lambda edge: self.cities_matrix[edge[0]][edge[1]], reverse=False) + for vertex_u, vertex_v in edges_list: + if disjoint_sets.find(vertex_v) != disjoint_sets.find(vertex_u): + forest_value += self.cities_matrix[vertex_u][vertex_v] + disjoint_sets.union(vertex_v, vertex_u) + return forest_value + + def init_distance_matrix(self, all_cities): + """ Generate matrix of distance between all cities """ + return {row_ind: {col_ind: distance(row_coords, col_coords) for col_ind, col_coords in all_cities.items()} + for row_ind, row_coords in all_cities.items()} + + +class DisjointSets: + """Union-find (disjoint-set) structure with union by rank, used to build the + MST for the TravelingSalesman heuristic.""" + + def __init__(self): + self.parent = {} + self.rank = {} + + def make_set(self, vertex): + """Create a new singleton set whose only element is vertex.""" + self.parent[vertex] = vertex + self.rank[vertex] = 0 + + def find(self, vertex): + """Return the representative (root) of the set that vertex belongs to.""" + if self.parent[vertex] == vertex: + return vertex + return self.find(self.parent[vertex]) + + def union(self, vertex_u, vertex_v): + """Merge the sets containing vertex_u and vertex_v, attaching the lower-rank + tree under the higher-rank one (union by rank).""" + vertex_u_root = self.find(vertex_u) + vertex_v_root = self.find(vertex_v) + + if self.rank[vertex_u_root] > self.rank[vertex_v_root]: + self.parent[vertex_v_root] = vertex_u_root + elif self.rank[vertex_u_root] < self.rank[vertex_v_root]: + self.parent[vertex_u_root] = vertex_v_root + else: + self.parent[vertex_u_root] = vertex_v_root + self.rank[vertex_v_root] += 1 + + +# ______________________________________________________________________________ + +class PlanRoute(Problem): + """ The problem of moving the Hybrid Wumpus Agent from one place to other """ + + def __init__(self, initial, goal, allowed, dimrow): + """ Define goal state and initialize a problem """ + super().__init__(initial, goal) + self.dimrow = dimrow + self.goal = goal + self.allowed = allowed + + def actions(self, state): + """ Return the actions that can be executed in the given state. + The result would be a list, since there are only three possible actions + in any given state of the environment """ + + possible_actions = ['Forward', 'TurnLeft', 'TurnRight'] + x, y = state.get_location() + orientation = state.get_orientation() + + # Prevent Bumps + if x == 1 and orientation == 'LEFT': + if 'Forward' in possible_actions: + possible_actions.remove('Forward') + if y == 1 and orientation == 'DOWN': + if 'Forward' in possible_actions: + possible_actions.remove('Forward') + if x == self.dimrow and orientation == 'RIGHT': + if 'Forward' in possible_actions: + possible_actions.remove('Forward') + if y == self.dimrow and orientation == 'UP': + if 'Forward' in possible_actions: + possible_actions.remove('Forward') + + return possible_actions + + def result(self, state, action): + """ Given state and action, return a new state that is the result of the action. + Action is assumed to be a valid action in the state. A fresh state object is + returned; the input state is never mutated, so the search tree stays consistent. """ + x, y = state.get_location() + orientation = state.get_orientation() + proposed_loc = None + new_orientation = orientation + + # Move Forward + if action == 'Forward': + if orientation == 'UP': + proposed_loc = [x, y + 1] + elif orientation == 'DOWN': + proposed_loc = [x, y - 1] + elif orientation == 'LEFT': + proposed_loc = [x - 1, y] + elif orientation == 'RIGHT': + proposed_loc = [x + 1, y] + else: + raise Exception('InvalidOrientation') + + # Rotate counter-clockwise + elif action == 'TurnLeft': + new_orientation = {'UP': 'LEFT', 'DOWN': 'RIGHT', 'LEFT': 'DOWN', 'RIGHT': 'UP'}[orientation] + + # Rotate clockwise + elif action == 'TurnRight': + new_orientation = {'UP': 'RIGHT', 'DOWN': 'LEFT', 'LEFT': 'UP', 'RIGHT': 'DOWN'}[orientation] + + new_x, new_y = x, y + if proposed_loc is not None and proposed_loc in self.allowed: + new_x, new_y = proposed_loc[0], proposed_loc[1] + + return type(state)(new_x, new_y, new_orientation) + + def goal_test(self, state): + """ Return True when the state matches any goal position. The goal is a + collection of [x, y] cells or of position objects; for the latter the + orientation must match too (e.g. a shooting position). """ + for goal in self.goal: + if hasattr(goal, 'get_location'): + if (state.get_location() == goal.get_location() and + state.get_orientation() == goal.get_orientation()): + return True + elif list(state.get_location()) == list(goal): + return True + return False + + def h(self, node): + """ Manhattan distance from the node's location to the nearest goal cell. """ + x1, y1 = node.state.get_location() + distances = [] + for goal in self.goal: + x2, y2 = goal.get_location() if hasattr(goal, 'get_location') else (goal[0], goal[1]) + distances.append(abs(x2 - x1) + abs(y2 - y1)) + return min(distances) if distances else 0 + + +# ______________________________________________________________________________ +# Other search algorithms + + +def recursive_best_first_search(problem, h=None): + """[Figure 3.26] A linear-space best-first search: it mimics A* but only keeps the + current path, backing up the best f-value of each forgotten subtree so it knows + where to resume.""" + h = memoize(h or problem.h, 'h') + + def RBFS(problem, node, flimit): + if problem.goal_test(node.state): + return node, 0 # (The second value is immaterial) + successors = node.expand(problem) + if len(successors) == 0: + return None, np.inf + for s in successors: + s.f = max(s.path_cost + h(s), node.f) + while True: + # Order by lowest f value + successors.sort(key=lambda x: x.f) + best = successors[0] + if best.f > flimit: + return None, best.f + if len(successors) > 1: + alternative = successors[1].f + else: + alternative = np.inf + result, best.f = RBFS(problem, best, min(flimit, alternative)) + if result is not None: + return result, best.f + + node = Node(problem.initial) + node.f = h(node) + result, bestf = RBFS(problem, node, np.inf) + return result + + +def hill_climbing(problem): + """ + [Figure 4.2] + From the initial node, keep choosing the neighbor with highest value, + stopping when no neighbor is better. + """ + current = Node(problem.initial) + while True: + neighbors = current.expand(problem) + if not neighbors: + break + neighbor = argmax_random_tie(neighbors, key=lambda node: problem.value(node.state)) + if problem.value(neighbor.state) <= problem.value(current.state): + break + current = neighbor + return current.state + + +# The following hill-climbing variants and local beam search are described in +# AIMA Section 4.1 (Local Search) but the book gives no pseudocode for them, so +# these are simple, faithful implementations of the prose descriptions. + + +def stochastic_hill_climbing(problem): + """Hill climbing that, instead of the steepest neighbor, picks *at random* + among the uphill neighbors (AIMA 4e Section 4.1.1 -- no pseudocode in the book). + """ + current = Node(problem.initial) + while True: + uphill = [node for node in current.expand(problem) + if problem.value(node.state) > problem.value(current.state)] + if not uphill: + break + current = random.choice(uphill) + return current.state + + +def first_choice_hill_climbing(problem, tries=100): + """Hill climbing that generates successors at random until it finds one better + than the current state -- useful when a state has very many successors (AIMA 4e + Section 4.1.1 -- no pseudocode in the book). + """ + current = Node(problem.initial) + while True: + neighbors = current.expand(problem) + if not neighbors: + break + better = None + for _ in range(tries): + candidate = random.choice(neighbors) + if problem.value(candidate.state) > problem.value(current.state): + better = candidate + break + if better is None: + break + current = better + return current.state + + +def random_restart_hill_climbing(problem, new_state, restarts=10): + """Run :func:`hill_climbing` from ``restarts`` random initial states (each from + the 0-argument ``new_state`` callable) and return the best state found (AIMA 4e + Section 4.1.1 -- no pseudocode in the book). Note: it sets ``problem.initial``. + """ + best = None + for _ in range(restarts): + problem.initial = new_state() + state = hill_climbing(problem) + if best is None or problem.value(state) > problem.value(best): + best = state + return best + + +def local_beam_search(problem, k=4, new_state=None, iterations=1000): + """Keep ``k`` states; at each step expand all of them and keep the ``k`` best + successors, stopping when none improves on the current best (AIMA 4e Section + 4.1.3 -- no pseudocode in the book). The initial ``k`` states come from + ``new_state`` (a 0-argument callable) or default to the problem's initial state. + """ + current = [Node(new_state() if new_state else problem.initial) for _ in range(k)] + for _ in range(iterations): + neighbors = [nb for node in current for nb in node.expand(problem)] + if not neighbors: + break + best = sorted(neighbors, key=lambda node: problem.value(node.state), reverse=True)[:k] + if problem.value(best[0].state) <= max(problem.value(node.state) for node in current): + break + current = best + return max(current, key=lambda node: problem.value(node.state)).state + + +def exp_schedule(k=20, lam=0.005, limit=100): + """One possible schedule function for simulated annealing""" + return lambda t: (k * np.exp(-lam * t) if t < limit else 0) + + +def simulated_annealing(problem, schedule=exp_schedule()): + """[Figure 4.5] CAUTION: This differs from the pseudocode as it + returns a state instead of a Node.""" + current = Node(problem.initial) + for t in range(sys.maxsize): + T = schedule(t) + if T == 0: + return current.state + neighbors = current.expand(problem) + if not neighbors: + return current.state + next_choice = random.choice(neighbors) + delta_e = problem.value(next_choice.state) - problem.value(current.state) + if delta_e > 0 or probability(np.exp(delta_e / T)): + current = next_choice + + +def simulated_annealing_full(problem, schedule=exp_schedule()): + """ This version returns all the states encountered in reaching + the goal state.""" + states = [] + current = Node(problem.initial) + for t in range(sys.maxsize): + states.append(current.state) + T = schedule(t) + if T == 0: + return states + neighbors = current.expand(problem) + if not neighbors: + return current.state + next_choice = random.choice(neighbors) + delta_e = problem.value(next_choice.state) - problem.value(current.state) + if delta_e > 0 or probability(np.exp(delta_e / T)): + current = next_choice + + +def and_or_graph_search(problem): + """[Figure 4.11]Used when the environment is nondeterministic and completely observable. + Contains OR nodes where the agent is free to choose any action. + After every action there is an AND node which contains all possible states + the agent may reach due to stochastic nature of environment. + The agent must be able to handle all possible states of the AND node (as it + may end up in any of them). + Returns a conditional plan to reach goal state, + or failure if the former is not possible.""" + + # functions used by and_or_search + def or_search(state, problem, path): + """returns a plan as a list of actions""" + if problem.goal_test(state): + return [] + if state in path: + return None + for action in problem.actions(state): + plan = and_search(problem.result(state, action), + problem, path + [state, ]) + if plan is not None: + return [action, plan] + + def and_search(states, problem, path): + """Returns plan in form of dictionary where we take action plan[s] if we reach state s.""" + plan = {} + for s in states: + plan[s] = or_search(s, problem, path) + if plan[s] is None: + return None + return plan + + # body of and or search + return or_search(problem.initial, problem, []) + + +# Pre-defined actions for PeakFindingProblem +directions4 = {'W': (-1, 0), 'N': (0, 1), 'E': (1, 0), 'S': (0, -1)} +directions8 = dict(directions4) +directions8.update({'NW': (-1, 1), 'NE': (1, 1), 'SE': (1, -1), 'SW': (-1, -1)}) + + +class PourProblem(Problem): + """Problem about pouring water between jugs to achieve some water level. + Each state is a tuple of levels. In the initialization, provide a tuple of + capacities, e.g. PourProblem(initial=(2, 4, 3), goals={7}, capacities=(8, 16, 32)), + which means three jugs of capacity 8, 16, 32 currently filled with 2, 4, 3 units + of water, respectively, and the goal is to get a level of 7 in any one of the jugs.""" + + def __init__(self, initial=None, goals=(), capacities=None): + super().__init__(initial, goals) + self.goals = goals + self.capacities = capacities + + def actions(self, state): + """The actions executable in this state: fill or dump any jug, or pour one into another.""" + jugs = range(len(state)) + return ([('Fill', i) for i in jugs if state[i] != self.capacities[i]] + + [('Dump', i) for i in jugs if state[i] != 0] + + [('Pour', i, j) for i in jugs for j in jugs if i != j]) + + def result(self, state, action): + """The state that results from executing this action in this state.""" + result = list(state) + act, i, j = action[0], action[1], action[-1] + if act == 'Fill': # fill jug i to its capacity + result[i] = self.capacities[i] + elif act == 'Dump': # empty jug i + result[i] = 0 + elif act == 'Pour': # pour jug i into jug j + a, b = state[i], state[j] + result[i], result[j] = ((0, a + b) if (a + b <= self.capacities[j]) + else (a + b - self.capacities[j], self.capacities[j])) + else: + raise ValueError('unknown action', action) + return tuple(result) + + def goal_test(self, state): + """True if any of the jugs has a level equal to one of the goal levels.""" + return any(level in self.goals for level in state) + + +class PeakFindingProblem(Problem): + """Problem of finding the highest peak in a limited grid""" + + def __init__(self, initial, grid, defined_actions=directions4): + """The grid is a 2 dimensional array/list whose state is specified by tuple of indices""" + super().__init__(initial) + self.grid = grid + self.defined_actions = defined_actions + self.n = len(grid) + assert self.n > 0 + self.m = len(grid[0]) + assert self.m > 0 + + def actions(self, state): + """Returns the list of actions which are allowed to be taken from the given state""" + allowed_actions = [] + for action in self.defined_actions: + next_state = vector_add(state, self.defined_actions[action]) + if 0 <= next_state[0] <= self.n - 1 and 0 <= next_state[1] <= self.m - 1: + allowed_actions.append(action) + + return allowed_actions + + def result(self, state, action): + """Moves in the direction specified by action""" + return vector_add(state, self.defined_actions[action]) + + def value(self, state): + """Value of a state is the value it is the index to""" + x, y = state + assert 0 <= x < self.n + assert 0 <= y < self.m + return self.grid[x][y] + + +class GridProblem(Problem): + """Shortest-path finding on a 2-D grid. + + States are ``(x, y)`` cells. The agent steps between in-bounds, obstacle-free + cells -- 4-connected via ``directions4`` by default, or pass ``directions8`` + for 8-connected movement. Every step costs 1; ``h`` is the Manhattan distance + to the goal (admissible for the 4-connected, unit-cost case). + """ + + def __init__(self, initial, goal, width, height, obstacles=(), directions=directions4): + super().__init__(initial, goal) + self.width = width + self.height = height + self.obstacles = set(map(tuple, obstacles)) + self.directions = directions + + def passable(self, cell): + """True if ``cell`` is on the grid and not an obstacle.""" + x, y = cell + return 0 <= x < self.width and 0 <= y < self.height and tuple(cell) not in self.obstacles + + def actions(self, state): + """The directions whose neighbouring cell is on the grid and obstacle-free.""" + return [a for a, d in self.directions.items() if self.passable(vector_add(state, d))] + + def result(self, state, action): + """Step one cell in ``action``'s direction.""" + return vector_add(state, self.directions[action]) + + def path_cost(self, c, state1, action, state2): + """Unit cost per step.""" + return c + 1 + + def h(self, node): + """Manhattan distance from ``node`` (or a state) to the goal.""" + x, y = node.state if isinstance(node, Node) else node + gx, gy = self.goal + return abs(x - gx) + abs(y - gy) + + +class OnlineDFSAgent: + """ + [Figure 4.21] + The abstract class for an OnlineDFSAgent. Override + update_state method to convert percept to state. While initializing + the subclass a problem needs to be provided which is an instance of + a subclass of the Problem class. + """ + + def __init__(self, problem): + self.problem = problem + self.s = None + self.a = None + self.untried = dict() + self.unbacktracked = dict() + self.result = {} + + def __call__(self, percept): + s1 = self.update_state(percept) + if self.problem.goal_test(s1): + self.a = None + else: + if s1 not in self.untried.keys(): + self.untried[s1] = list(self.problem.actions(s1)) + self.unbacktracked[s1] = [] + if self.s is not None: + self.result[(self.s, self.a)] = s1 + self.unbacktracked[s1].insert(0, self.s) + if len(self.untried[s1]) == 0: + if len(self.unbacktracked[s1]) == 0: + self.a = None + else: + # else a <- an action b such that result[s', b] = POP(unbacktracked[s']) + unbacktracked_pop = self.unbacktracked[s1].pop(0) + for (s, b) in self.result.keys(): + if s == s1 and self.result[(s, b)] == unbacktracked_pop: + self.a = b + break + else: + self.a = self.untried[s1].pop() + self.s = s1 + return self.a + + def update_state(self, percept): + """To be overridden in most cases. The default case + assumes the percept to be of type state.""" + return percept + + +# ______________________________________________________________________________ + + +class OnlineSearchProblem(Problem): + """ + A problem which is solved by an agent executing + actions, rather than by just computation. + Carried in a deterministic and a fully observable environment.""" + + def __init__(self, initial, goal, graph): + super().__init__(initial, goal) + self.graph = graph + + def actions(self, state): + """Return the actions available from state (the outgoing graph edges).""" + return self.graph.graph_dict[state].keys() + + def output(self, state, action): + """Return the state reached by taking action in state.""" + return self.graph.graph_dict[state][action] + + def h(self, state): + """Returns least possible cost to reach a goal for the given state.""" + return self.graph.least_costs[state] + + def c(self, s, a, s1): + """Returns a cost estimate for an agent to move from state 's' to state 's1'.""" + return 1 + + def update_state(self, percept): + """Convert a percept into a state (abstract; override in subclasses).""" + raise NotImplementedError + + def goal_test(self, state): + """Return True if state is the goal state.""" + if state == self.goal: + return True + return False + + +class LRTAStarAgent: + """ [Figure 4.24] + Abstract class for LRTA*-Agent. A problem needs to be + provided which is an instance of a subclass of Problem Class. + + Takes a OnlineSearchProblem [Figure 4.23] as a problem. + """ + + def __init__(self, problem): + self.problem = problem + # self.result = {} # no need as we are using problem.result + self.H = {} + self.s = None + self.a = None + + def __call__(self, s1): # as of now s1 is a state rather than a percept + if self.problem.goal_test(s1): + self.a = None + return self.a + else: + if s1 not in self.H: + self.H[s1] = self.problem.h(s1) + if self.s is not None: + # self.result[(self.s, self.a)] = s1 # no need as we are using problem.output + + # minimum cost for action b in problem.actions(s) + self.H[self.s] = min(self.LRTA_cost(self.s, b, self.problem.output(self.s, b), + self.H) for b in self.problem.actions(self.s)) + + # an action b in problem.actions(s1) that minimizes costs + self.a = min(self.problem.actions(s1), + key=lambda b: self.LRTA_cost(s1, b, self.problem.output(s1, b), self.H)) + + self.s = s1 + return self.a + + def LRTA_cost(self, s, a, s1, H): + """Returns cost to move from state 's' to state 's1' plus + estimated cost to get to goal from s1.""" + print(s, a, s1) + if s1 is None: + return self.problem.h(s) + else: + # sometimes we need to get H[s1] which we haven't yet added to H + # to replace this try, except: we can initialize H with values from problem.h + try: + return self.problem.c(s, a, s1) + self.H[s1] + except: + return self.problem.c(s, a, s1) + self.problem.h(s1) + + +# ______________________________________________________________________________ +# Genetic Algorithm + + +def genetic_search(problem, ngen=1000, pmut=0.1, n=20): + """Call genetic_algorithm on the appropriate parts of a problem. + This requires the problem to have states that can mate and mutate, + plus a value method that scores states.""" + + # NOTE: This is not tested and might not work. + # TODO: Use this function to make Problems work with genetic_algorithm. + + s = problem.initial_state + states = [problem.result(s, a) for a in problem.actions(s)] + random.shuffle(states) + return genetic_algorithm(states[:n], problem.value, ngen, pmut) + + +def genetic_algorithm(population, fitness_fn, gene_pool=[0, 1], f_thres=None, ngen=1000, pmut=0.1): + """[Figure 4.8]""" + for i in range(ngen): + population = [mutate(recombine(*select(2, population, fitness_fn)), gene_pool, pmut) + for i in range(len(population))] + + fittest_individual = fitness_threshold(fitness_fn, f_thres, population) + if fittest_individual: + return fittest_individual + + return max(population, key=fitness_fn) + + +def fitness_threshold(fitness_fn, f_thres, population): + """Return the fittest individual if its fitness reaches f_thres, otherwise None + (also None when no threshold f_thres is given).""" + if not f_thres: + return None + + fittest_individual = max(population, key=fitness_fn) + if fitness_fn(fittest_individual) >= f_thres: + return fittest_individual + + return None + + +def init_population(pop_number, gene_pool, state_length): + """Initializes population for genetic algorithm + pop_number : Number of individuals in population + gene_pool : List of possible values for individuals + state_length: The length of each individual""" + g = len(gene_pool) + population = [] + for i in range(pop_number): + new_individual = [gene_pool[random.randrange(0, g)] for j in range(state_length)] + population.append(new_individual) + + return population + + +def select(r, population, fitness_fn): + """Select r individuals from the population, sampling each with probability + proportional to its fitness.""" + fitnesses = map(fitness_fn, population) + sampler = weighted_sampler(population, fitnesses) + return [sampler() for i in range(r)] + + +def recombine(x, y): + """Single-point crossover: combine a random-length prefix of x with the + corresponding suffix of y.""" + n = len(x) + c = random.randrange(0, n) + return x[:c] + y[c:] + + +def recombine_uniform(x, y): + """Uniform crossover: build a child string by taking roughly half of its genes + from x and the rest from y, at random positions.""" + n = len(x) + result = [0] * n + indexes = random.sample(range(n), n) + for i in range(n): + ix = indexes[i] + result[ix] = x[ix] if i < n / 2 else y[ix] + + return ''.join(str(r) for r in result) + + +def mutate(x, gene_pool, pmut): + """With probability pmut, replace one randomly chosen gene of x with a random + gene from gene_pool; otherwise return x unchanged.""" + if random.uniform(0, 1) >= pmut: + return x + + n = len(x) + g = len(gene_pool) + c = random.randrange(0, n) + r = random.randrange(0, g) + + new_gene = gene_pool[r] + return x[:c] + [new_gene] + x[c + 1:] + + +# _____________________________________________________________________________ +# The remainder of this file implements examples for the search algorithms. + +# ______________________________________________________________________________ +# Graphs and Graph Problems + + +class Graph: + """A graph connects nodes (vertices) by edges (links). Each edge can also + have a length associated with it. The constructor call is something like:: + + g = Graph({'A': {'B': 1, 'C': 2}) + + this makes a graph with 3 nodes, A, B, and C, with an edge of length 1 from + A to B, and an edge of length 2 from A to C. You can also do:: + + g = Graph({'A': {'B': 1, 'C': 2}, directed=False) + + This makes an undirected graph, so inverse links are also added. The graph + stays undirected; if you add more links with g.connect('B', 'C', 3), then + inverse link is also added. You can use g.nodes() to get a list of nodes, + g.get('A') to get a dict of links out of A, and g.get('A', 'B') to get the + length of the link from A to B. 'Lengths' can actually be any object at + all, and nodes can be any hashable object.""" + + def __init__(self, graph_dict=None, directed=True): + self.graph_dict = graph_dict or {} + self.directed = directed + if not directed: + self.make_undirected() + + def make_undirected(self): + """Make a digraph into an undirected graph by adding symmetric edges.""" + for a in list(self.graph_dict.keys()): + for (b, dist) in self.graph_dict[a].items(): + self.connect1(b, a, dist) + + def connect(self, A, B, distance=1): + """Add a link from A and B of given distance, and also add the inverse + link if the graph is undirected.""" + self.connect1(A, B, distance) + if not self.directed: + self.connect1(B, A, distance) + + def connect1(self, A, B, distance): + """Add a link from A to B of given distance, in one direction only.""" + self.graph_dict.setdefault(A, {})[B] = distance + + def get(self, a, b=None): + """Return a link distance or a dict of {node: distance} entries. + .get(a,b) returns the distance or None; + .get(a) returns a dict of {node: distance} entries, possibly {}.""" + links = self.graph_dict.setdefault(a, {}) + if b is None: + return links + else: + return links.get(b) + + def nodes(self): + """Return a list of nodes in the graph.""" + s1 = set([k for k in self.graph_dict.keys()]) + s2 = set([k2 for v in self.graph_dict.values() for k2, v2 in v.items()]) + nodes = s1.union(s2) + return list(nodes) + + +def UndirectedGraph(graph_dict=None): + """Build a Graph where every edge (including future ones) goes both ways.""" + return Graph(graph_dict=graph_dict, directed=False) + + +def RandomGraph(nodes=list(range(10)), min_links=2, width=400, height=300, + curvature=lambda: random.uniform(1.1, 1.5)): + """Construct a random graph, with the specified nodes, and random links. + The nodes are laid out randomly on a (width x height) rectangle. + Then each node is connected to the min_links nearest neighbors. + Because inverse links are added, some nodes will have more connections. + The distance between nodes is the hypotenuse times curvature(), + where curvature() defaults to a random number between 1.1 and 1.5.""" + g = UndirectedGraph() + g.locations = {} + # Build the cities + for node in nodes: + g.locations[node] = (random.randrange(width), random.randrange(height)) + # Build roads from each city to at least min_links nearest neighbors. + for i in range(min_links): + for node in nodes: + if len(g.get(node)) < min_links: + here = g.locations[node] + + def distance_to_node(n): + if n is node or g.get(node, n): + return np.inf + return distance(g.locations[n], here) + + neighbor = min(nodes, key=distance_to_node) + d = distance(g.locations[neighbor], here) * curvature() + g.connect(node, neighbor, int(d)) + return g + + +""" [Figure 3.2] +Simplified road map of Romania +""" +romania_map = UndirectedGraph(dict( + Arad=dict(Zerind=75, Sibiu=140, Timisoara=118), + Bucharest=dict(Urziceni=85, Pitesti=101, Giurgiu=90, Fagaras=211), + Craiova=dict(Drobeta=120, Rimnicu=146, Pitesti=138), + Drobeta=dict(Mehadia=75), + Eforie=dict(Hirsova=86), + Fagaras=dict(Sibiu=99), + Hirsova=dict(Urziceni=98), + Iasi=dict(Vaslui=92, Neamt=87), + Lugoj=dict(Timisoara=111, Mehadia=70), + Oradea=dict(Zerind=71, Sibiu=151), + Pitesti=dict(Rimnicu=97), + Rimnicu=dict(Sibiu=80), + Urziceni=dict(Vaslui=142))) +romania_map.locations = dict( + Arad=(91, 492), Bucharest=(400, 327), Craiova=(253, 288), + Drobeta=(165, 299), Eforie=(562, 293), Fagaras=(305, 449), + Giurgiu=(375, 270), Hirsova=(534, 350), Iasi=(473, 506), + Lugoj=(165, 379), Mehadia=(168, 339), Neamt=(406, 537), + Oradea=(131, 571), Pitesti=(320, 368), Rimnicu=(233, 410), + Sibiu=(207, 457), Timisoara=(94, 410), Urziceni=(456, 350), + Vaslui=(509, 444), Zerind=(108, 531)) + +""" [Figure 4.9] +Eight possible states of the vacumm world +Each state is represented as + * "State of the left room" "State of the right room" "Room in which the agent + is present" +1 - DDL Dirty Dirty Left +2 - DDR Dirty Dirty Right +3 - DCL Dirty Clean Left +4 - DCR Dirty Clean Right +5 - CDL Clean Dirty Left +6 - CDR Clean Dirty Right +7 - CCL Clean Clean Left +8 - CCR Clean Clean Right +""" +vacuum_world = Graph(dict( + State_1=dict(Suck=['State_7', 'State_5'], Right=['State_2']), + State_2=dict(Suck=['State_8', 'State_4'], Left=['State_2']), + State_3=dict(Suck=['State_7'], Right=['State_4']), + State_4=dict(Suck=['State_4', 'State_2'], Left=['State_3']), + State_5=dict(Suck=['State_5', 'State_1'], Right=['State_6']), + State_6=dict(Suck=['State_8'], Left=['State_5']), + State_7=dict(Suck=['State_7', 'State_3'], Right=['State_8']), + State_8=dict(Suck=['State_8', 'State_6'], Left=['State_7']) +)) + +""" [Figure 4.23] +One-dimensional state space Graph +""" +one_dim_state_space = Graph(dict( + State_1=dict(Right='State_2'), + State_2=dict(Right='State_3', Left='State_1'), + State_3=dict(Right='State_4', Left='State_2'), + State_4=dict(Right='State_5', Left='State_3'), + State_5=dict(Right='State_6', Left='State_4'), + State_6=dict(Left='State_5') +)) +one_dim_state_space.least_costs = dict( + State_1=8, + State_2=9, + State_3=2, + State_4=2, + State_5=4, + State_6=3) + +""" [Figure 6.1] +Principal states and territories of Australia +""" +australia_map = UndirectedGraph(dict( + T=dict(), + SA=dict(WA=1, NT=1, Q=1, NSW=1, V=1), + NT=dict(WA=1, Q=1), + NSW=dict(Q=1, V=1))) +australia_map.locations = dict(WA=(120, 24), NT=(135, 20), SA=(135, 30), + Q=(145, 20), NSW=(145, 32), T=(145, 42), + V=(145, 37)) + + +class GraphProblem(Problem): + """The problem of searching a graph from one node to another.""" + + def __init__(self, initial, goal, graph): + super().__init__(initial, goal) + self.graph = graph + + def actions(self, A): + """The actions at a graph node are just its neighbors.""" + return list(self.graph.get(A).keys()) + + def result(self, state, action): + """The result of going to a neighbor is just that neighbor.""" + return action + + def path_cost(self, cost_so_far, A, action, B): + """Cost so far plus the length of the edge from A to B (np.inf if no edge).""" + return cost_so_far + (self.graph.get(A, B) or np.inf) + + def find_min_edge(self): + """Find minimum value of edges.""" + m = np.inf + for d in self.graph.graph_dict.values(): + local_min = min(d.values()) + m = min(m, local_min) + + return m + + def h(self, node): + """h function is straight-line distance from a node's state to goal.""" + locs = getattr(self.graph, 'locations', None) + if locs: + if type(node) is str: + return int(distance(locs[node], locs[self.goal])) + + return int(distance(locs[node.state], locs[self.goal])) + else: + return np.inf + + +class GraphProblemStochastic(GraphProblem): + """ + A version of GraphProblem where an action can lead to + nondeterministic output i.e. multiple possible states. + + Define the graph as dict(A = dict(Action = [[, , ...], ], ...), ...) + A the dictionary format is different, make sure the graph is created as a directed graph. + """ + + def result(self, state, action): + """Return the list of possible states that the action may lead to.""" + return self.graph.get(state, action) + + def path_cost(self): + """Not defined for the stochastic graph problem.""" + raise NotImplementedError + + +# ______________________________________________________________________________ + + +class NQueensProblem(Problem): + """The problem of placing N queens on an NxN board with none attacking + each other. A state is represented as an N-element array, where + a value of r in the c-th entry means there is a queen at column c, + row r, and a value of -1 means that the c-th column has not been + filled in yet. We fill in columns left to right. + >>> depth_first_tree_search(NQueensProblem(8)) + + """ + + def __init__(self, N): + super().__init__(tuple([-1] * N)) + self.N = N + + def actions(self, state): + """In the leftmost empty column, try all non-conflicting rows.""" + if state[-1] != -1: + return [] # All columns filled; no successors + else: + col = state.index(-1) + return [row for row in range(self.N) + if not self.conflicted(state, row, col)] + + def result(self, state, row): + """Place the next queen at the given row.""" + col = state.index(-1) + new = list(state[:]) + new[col] = row + return tuple(new) + + def conflicted(self, state, row, col): + """Would placing a queen at (row, col) conflict with anything?""" + return any(self.conflict(row, col, state[c], c) + for c in range(col)) + + def conflict(self, row1, col1, row2, col2): + """Would putting two queens in (row1, col1) and (row2, col2) conflict?""" + return (row1 == row2 or # same row + col1 == col2 or # same column + row1 - col1 == row2 - col2 or # same \ diagonal + row1 + col1 == row2 + col2) # same / diagonal + + def goal_test(self, state): + """Check if all columns filled, no conflicts.""" + if state[-1] == -1: + return False + return not any(self.conflicted(state, state[col], col) + for col in range(len(state))) + + def h(self, node): + """Return number of conflicting queens for a given node""" + num_conflicts = 0 + for (r1, c1) in enumerate(node.state): + for (r2, c2) in enumerate(node.state): + if (r1, c1) != (r2, c2): + num_conflicts += self.conflict(r1, c1, r2, c2) + + return num_conflicts + + +# ______________________________________________________________________________ +# Inverse Boggle: Search for a high-scoring Boggle board. A good domain for +# iterative-repair and related search techniques, as suggested by Justin Boyan. + + +ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + +cubes16 = ['FORIXB', 'MOQABJ', 'GURILW', 'SETUPL', + 'CMPDAE', 'ACITAO', 'SLCRAE', 'ROMASH', + 'NODESW', 'HEFIYE', 'ONUDTK', 'TEVIGN', + 'ANEDVZ', 'PINESH', 'ABILYT', 'GKYLEU'] + + +def random_boggle(n=4): + """Return a random Boggle board of size n x n. + We represent a board as a linear list of letters.""" + cubes = [cubes16[i % 16] for i in range(n * n)] + random.shuffle(cubes) + return list(map(random.choice, cubes)) + + +# The best 5x5 board found by Boyan, with our word list this board scores +# 2274 words, for a score of 9837 + + +boyan_best = list('RSTCSDEIAEGNLRPEATESMSSID') + + +def print_boggle(board): + """Print the board in a 2-d array.""" + n2 = len(board) + n = exact_sqrt(n2) + for i in range(n2): + + if i % n == 0 and i > 0: + print() + if board[i] == 'Q': + print('Qu', end=' ') + else: + print(str(board[i]) + ' ', end=' ') + print() + + +def boggle_neighbors(n2, cache={}): + """Return a list of lists, where the i-th element is the list of indexes + for the neighbors of square i.""" + if cache.get(n2): + return cache.get(n2) + n = exact_sqrt(n2) + neighbors = [None] * n2 + for i in range(n2): + neighbors[i] = [] + on_top = i < n + on_bottom = i >= n2 - n + on_left = i % n == 0 + on_right = (i + 1) % n == 0 + if not on_top: + neighbors[i].append(i - n) + if not on_left: + neighbors[i].append(i - n - 1) + if not on_right: + neighbors[i].append(i - n + 1) + if not on_bottom: + neighbors[i].append(i + n) + if not on_left: + neighbors[i].append(i + n - 1) + if not on_right: + neighbors[i].append(i + n + 1) + if not on_left: + neighbors[i].append(i - 1) + if not on_right: + neighbors[i].append(i + 1) + cache[n2] = neighbors + return neighbors + + +def exact_sqrt(n2): + """If n2 is a perfect square, return its square root, else raise error.""" + n = int(np.sqrt(n2)) + assert n * n == n2 + return n + + +# _____________________________________________________________________________ + + +class Wordlist: + """This class holds a list of words. You can use (word in wordlist) + to check if a word is in the list, or wordlist.lookup(prefix) + to see if prefix starts any of the words in the list.""" + + def __init__(self, file, min_len=3): + lines = file.read().upper().split() + self.words = [word for word in lines if len(word) >= min_len] + self.words.sort() + self.bounds = {} + for c in ALPHABET: + c2 = chr(ord(c) + 1) + self.bounds[c] = (bisect.bisect(self.words, c), + bisect.bisect(self.words, c2)) + + def lookup(self, prefix, lo=0, hi=None): + """See if prefix is in dictionary, as a full word or as a prefix. + Return two values: the first is the lowest i such that + words[i].startswith(prefix), or is None; the second is + True iff prefix itself is in the Wordlist.""" + words = self.words + if hi is None: + hi = len(words) + i = bisect.bisect_left(words, prefix, lo, hi) + if i < len(words) and words[i].startswith(prefix): + return i, (words[i] == prefix) + else: + return None, False + + def __contains__(self, word): + return self.lookup(word)[1] + + def __len__(self): + return len(self.words) + + +# _____________________________________________________________________________ + + +class BoggleFinder: + """A class that allows you to find all the words in a Boggle board.""" + + wordlist = None # A class variable, holding a wordlist + + def __init__(self, board=None): + if BoggleFinder.wordlist is None: + BoggleFinder.wordlist = Wordlist(open_data("EN-text/wordlist.txt")) + self.found = {} + if board: + self.set_board(board) + + def set_board(self, board=None): + """Set the board, and find all the words in it.""" + if board is None: + board = random_boggle() + self.board = board + self.neighbors = boggle_neighbors(len(board)) + self.found = {} + for i in range(len(board)): + lo, hi = self.wordlist.bounds[board[i]] + self.find(lo, hi, i, [], '') + return self + + def find(self, lo, hi, i, visited, prefix): + """Looking in square i, find the words that continue the prefix, + considering the entries in self.wordlist.words[lo:hi], and not + revisiting the squares in visited.""" + if i in visited: + return + wordpos, is_word = self.wordlist.lookup(prefix, lo, hi) + if wordpos is not None: + if is_word: + self.found[prefix] = True + visited.append(i) + c = self.board[i] + if c == 'Q': + c = 'QU' + prefix += c + for j in self.neighbors[i]: + self.find(wordpos, hi, j, visited, prefix) + visited.pop() + + def words(self): + """The words found.""" + return list(self.found.keys()) + + scores = [0, 0, 0, 0, 1, 2, 3, 5] + [11] * 100 + + def score(self): + """The total score for the words found, according to the rules.""" + return sum([self.scores[len(w)] for w in self.words()]) + + def __len__(self): + """The number of words found.""" + return len(self.found) + + +# _____________________________________________________________________________ + + +def boggle_hill_climbing(board=None, ntimes=100, verbose=True): + """Solve inverse Boggle by hill-climbing: find a high-scoring board by + starting with a random one and changing it.""" + finder = BoggleFinder() + if board is None: + board = random_boggle() + best = len(finder.set_board(board)) + for _ in range(ntimes): + i, oldc = mutate_boggle(board) + new = len(finder.set_board(board)) + if new > best: + best = new + if verbose: + print(best, _, board) + else: + board[i] = oldc # Change back + if verbose: + print_boggle(board) + return board, best + + +def mutate_boggle(board): + """Randomly replace one cell of a Boggle board with a random face of a random + die, returning the changed index and the old letter so the move can be undone.""" + i = random.randrange(len(board)) + oldc = board[i] + # random.choice(boyan_best) + board[i] = random.choice(random.choice(cubes16)) + return i, oldc + + +# ______________________________________________________________________________ + +# Code to compare searchers on various problems. + + +class InstrumentedProblem(Problem): + """Delegates to a problem, and keeps statistics.""" + + def __init__(self, problem): + self.problem = problem + self.succs = self.goal_tests = self.states = 0 + self.found = None + + def actions(self, state): + """Delegate to the wrapped problem, counting the successor query.""" + self.succs += 1 + return self.problem.actions(state) + + def result(self, state, action): + """Delegate to the wrapped problem, counting the generated state.""" + self.states += 1 + return self.problem.result(state, action) + + def goal_test(self, state): + """Delegate to the wrapped problem, counting the goal test and recording + the state when it is a goal.""" + self.goal_tests += 1 + result = self.problem.goal_test(state) + if result: + self.found = state + return result + + def path_cost(self, c, state1, action, state2): + """Delegate the path cost computation to the wrapped problem.""" + return self.problem.path_cost(c, state1, action, state2) + + def value(self, state): + """Delegate the state value computation to the wrapped problem.""" + return self.problem.value(state) + + def __getattr__(self, attr): + return getattr(self.problem, attr) + + def __repr__(self): + return '<{:4d}/{:4d}/{:4d}/{}>'.format(self.succs, self.goal_tests, + self.states, str(self.found)[:4]) + + +def compare_searchers(problems, header, + searchers=[breadth_first_tree_search, + breadth_first_graph_search, + depth_first_graph_search, + iterative_deepening_search, + depth_limited_search, + recursive_best_first_search]): + """Run every searcher on every problem and print a table of the resulting + InstrumentedProblem statistics (successors / goal tests / states generated).""" + + def do(searcher, problem): + p = InstrumentedProblem(problem) + searcher(p) + return p + + table = [[name(s)] + [do(s, p) for p in problems] for s in searchers] + print_table(table, header) + + +def compare_graph_searchers(): + """Prints a table of search results.""" + compare_searchers(problems=[GraphProblem('Arad', 'Bucharest', romania_map), + GraphProblem('Oradea', 'Neamt', romania_map), + GraphProblem('Q', 'WA', australia_map)], + header=['Searcher', 'romania_map(Arad, Bucharest)', + 'romania_map(Oradea, Neamt)', 'australia_map']) diff --git a/q-learning-9b/aima/text.py b/q-learning-9b/aima/text.py new file mode 100644 index 0000000..1bf693f --- /dev/null +++ b/q-learning-9b/aima/text.py @@ -0,0 +1,435 @@ +""" +Statistical Language Processing tools (Chapter 22) + +We define Unigram and Ngram text models, use them to generate random text, +and show the Viterbi algorithm for segmentation of letters into words. +Then we show a very simple Information Retrieval system, and an example +working on a tiny sample of Unix manual pages. +""" + +import heapq +import os +import re +from collections import defaultdict + +import numpy as np + +from aima import search +from aima.probabilistic_learning import CountingProbDist +from aima.utils import hashabledict + + +class UnigramWordModel(CountingProbDist): + """This is a discrete probability distribution over words, so you + can add, sample, or get P[word], just like with CountingProbDist. You can + also generate a random text, n words long, with P.samples(n).""" + + def __init__(self, observations, default=0): + # Call CountingProbDist constructor, + # passing the observations and default parameters. + super(UnigramWordModel, self).__init__(observations, default) + + def samples(self, n): + """Return a string of n words, random according to the model.""" + return ' '.join(self.sample() for i in range(n)) + + +class NgramWordModel(CountingProbDist): + """This is a discrete probability distribution over n-tuples of words. + You can add, sample or get P[(word1, ..., wordn)]. The method P.samples(n) + builds up an n-word sequence; P.add_cond_prob and P.add_sequence add data.""" + + def __init__(self, n, observation_sequence=None, default=0): + # In addition to the dictionary of n-tuples, cond_prob is a + # mapping from (w1, ..., wn-1) to P(wn | w1, ... wn-1) + CountingProbDist.__init__(self, default=default) + self.n = n + self.cond_prob = defaultdict() + self.add_sequence(observation_sequence or []) + + # __getitem__, top, sample inherited from CountingProbDist + # Note that they deal with tuples, not strings, as inputs + + def add_cond_prob(self, ngram): + """Build the conditional probabilities P(wn | (w1, ..., wn-1)""" + if ngram[:-1] not in self.cond_prob: + self.cond_prob[ngram[:-1]] = CountingProbDist() + self.cond_prob[ngram[:-1]].add(ngram[-1]) + + def add_sequence(self, words): + """Add each tuple words[i:i+n], using a sliding window.""" + n = self.n + + for i in range(len(words) - n + 1): + t = tuple(words[i:i + n]) + self.add(t) + self.add_cond_prob(t) + + def samples(self, nwords): + """Generate an n-word sentence by picking random samples + according to the model. At first pick a random n-gram and + from then on keep picking a character according to + P(c|wl-1, wl-2, ..., wl-n+1) where wl-1 ... wl-n+1 are the + last n - 1 words in the generated sentence so far.""" + n = self.n + output = list(self.sample()) + + for i in range(n, nwords): + last = output[-n + 1:] + next_word = self.cond_prob[tuple(last)].sample() + output.append(next_word) + + return ' '.join(output) + + +class NgramCharModel(NgramWordModel): + """An n-gram model over characters rather than words, trained on the + character n-grams of each word (prefixed with a space marker).""" + + def add_sequence(self, words): + """Add an empty space to every word to catch the beginning of words.""" + for word in words: + super().add_sequence(' ' + word) + + +class UnigramCharModel(NgramCharModel): + """A unigram (single-character) probability model: the frequency + distribution of individual characters across the observed words.""" + + def __init__(self, observation_sequence=None, default=0): + CountingProbDist.__init__(self, default=default) + self.n = 1 + self.cond_prob = defaultdict() + self.add_sequence(observation_sequence or []) + + def add_sequence(self, words): + """Count every character of every word in ``words`` into the model.""" + for word in words: + for char in word: + self.add(char) + + +# ______________________________________________________________________________ + + +def viterbi_segment(text, P): + """Find the best segmentation of the string of characters, given the + UnigramWordModel P.""" + # best[i] = best probability for text[0:i] + # words[i] = best word ending at position i + n = len(text) + words = [''] + list(text) + best = [1.0] + [0.0] * n + # Fill in the vectors best words via dynamic programming + for i in range(n + 1): + for j in range(0, i): + w = text[j:i] + curr_score = P[w] * best[i - len(w)] + if curr_score >= best[i]: + best[i] = curr_score + words[i] = w + # Now recover the sequence of best words + sequence = [] + i = len(words) - 1 + while i > 0: + sequence[0:0] = [words[i]] + i = i - len(words[i]) + # Return sequence of best words and overall probability + return sequence, best[-1] + + +# ______________________________________________________________________________ + + +# TODO(tmrts): Expose raw index +class IRSystem: + """A very simple Information Retrieval System, as discussed in Sect. 23.2. + The constructor s = IRSystem('the a') builds an empty system with two + stopwords. Next, index several documents with s.index_document(text, url). + Then ask queries with s.query('query words', n) to retrieve the top n + matching documents. Queries are literal words from the document, + except that stopwords are ignored, and there is one special syntax: + The query "learn: man cat", for example, runs "man cat" and indexes it.""" + + def __init__(self, stopwords='the a of'): + """Create an IR System. Optionally specify stopwords.""" + # index is a map of {word: {docid: count}}, where docid is an int, + # indicating the index into the documents list. + self.index = defaultdict(lambda: defaultdict(int)) + self.stopwords = set(words(stopwords)) + self.documents = [] + + def index_collection(self, filenames): + """Index a whole collection of files.""" + prefix = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + for filename in filenames: + self.index_document(open(filename).read(), os.path.relpath(filename, prefix)) + + def index_document(self, text, url): + """Index the text of a document.""" + # For now, use first line for title + title = text[:text.index('\n')].strip() + docwords = words(text) + docid = len(self.documents) + self.documents.append(Document(title, url, len(docwords))) + for word in docwords: + if word not in self.stopwords: + self.index[word][docid] += 1 + + def query(self, query_text, n=10): + """Return a list of n (score, docid) pairs for the best matches. + Also handle the special syntax for 'learn: command'.""" + if query_text.startswith("learn:"): + doctext = os.popen(query_text[len("learn:"):], 'r').read() + self.index_document(doctext, query_text) + return [] + + qwords = [w for w in words(query_text) if w not in self.stopwords] + shortest = min(qwords, key=lambda w: len(self.index[w])) + docids = self.index[shortest] + return heapq.nlargest(n, ((self.total_score(qwords, docid), docid) for docid in docids)) + + def score(self, word, docid): + """Compute a score for this word on the document with this docid.""" + # There are many options; here we take a very simple approach + return np.log(1 + self.index[word][docid]) / np.log(1 + self.documents[docid].nwords) + + def total_score(self, words, docid): + """Compute the sum of the scores of these words on the document with this docid.""" + return sum(self.score(word, docid) for word in words) + + def present(self, results): + """Present the results as a list.""" + for (score, docid) in results: + doc = self.documents[docid] + print("{:5.2}|{:25} | {}".format(100 * score, doc.url, doc.title[:45].expandtabs())) + + def present_results(self, query_text, n=10): + """Get results for the query and present them.""" + self.present(self.query(query_text, n)) + + +class UnixConsultant(IRSystem): + """A trivial IR system over a small collection of Unix man pages.""" + + def __init__(self): + IRSystem.__init__(self, stopwords="how do i the a of") + + import os + aima_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + mandir = os.path.join(aima_root, 'aima-data/MAN/') + man_files = [mandir + f for f in os.listdir(mandir) if f.endswith('.txt')] + + self.index_collection(man_files) + + +class Document: + """Metadata for a document: title and url; maybe add others later.""" + + def __init__(self, title, url, nwords): + self.title = title + self.url = url + self.nwords = nwords + + +def words(text, reg=re.compile('[a-z0-9]+')): + """Return a list of the words in text, ignoring punctuation and + converting everything to lowercase (to canonicalize). + + >>> words("``EGAD!'' Edgar cried.") + ['egad', 'edgar', 'cried'] + """ + return reg.findall(text.lower()) + + +def canonicalize(text): + """Return a canonical text: only lowercase letters and blanks. + + >>> canonicalize("``EGAD!'' Edgar cried.") + 'egad edgar cried' + """ + return ' '.join(words(text)) + + +# ______________________________________________________________________________ + +# Example application (not in book): decode a cipher. +# A cipher is a code that substitutes one character for another. +# A shift cipher is a rotation of the letters in the alphabet, +# such as the famous rot13, which maps A to N, B to M, etc. + +alphabet = 'abcdefghijklmnopqrstuvwxyz' + + +# Encoding + + +def shift_encode(plaintext, n): + """Encode text with a shift cipher that moves each letter up by n letters. + >>> shift_encode('abc z', 1) + 'bcd a' + """ + return encode(plaintext, alphabet[n:] + alphabet[:n]) + + +def rot13(plaintext): + """Encode text by rotating letters by 13 spaces in the alphabet. + >>> rot13('hello') + 'uryyb' + >>> rot13(rot13('hello')) + 'hello' + """ + return shift_encode(plaintext, 13) + + +def translate(plaintext, function): + """Translate chars of a plaintext with the given function.""" + result = "" + for char in plaintext: + result += function(char) + return result + + +def maketrans(from_, to_): + """Create a translation table and return the proper function.""" + trans_table = {} + for n, char in enumerate(from_): + trans_table[char] = to_[n] + + return lambda char: trans_table.get(char, char) + + +def encode(plaintext, code): + """Encode text using a code which is a permutation of the alphabet.""" + trans = maketrans(alphabet + alphabet.upper(), code + code.upper()) + + return translate(plaintext, trans) + + +def bigrams(text): + """Return a list of pairs in text (a sequence of letters or words). + >>> bigrams('this') + ['th', 'hi', 'is'] + >>> bigrams(['this', 'is', 'a', 'test']) + [['this', 'is'], ['is', 'a'], ['a', 'test']] + """ + return [text[i:i + 2] for i in range(len(text) - 1)] + + +# Decoding a Shift (or Caesar) Cipher + + +class ShiftDecoder: + """There are only 26 possible encodings, so we can try all of them, + and return the one with the highest probability, according to a + bigram probability distribution.""" + + def __init__(self, training_text): + training_text = canonicalize(training_text) + self.P2 = CountingProbDist(bigrams(training_text), default=1) + + def score(self, plaintext): + """Return a score for text based on how common letters pairs are.""" + + s = 1.0 + for bi in bigrams(plaintext): + s = s * self.P2[bi] + + return s + + def decode(self, ciphertext): + """Return the shift decoding of text with the best score.""" + + return max(all_shifts(ciphertext), key=lambda shift: self.score(shift)) + + +def all_shifts(text): + """Return a list of all 26 possible encodings of text by a shift cipher.""" + + yield from (shift_encode(text, i) for i, _ in enumerate(alphabet)) + + +# Decoding a General Permutation Cipher + + +class PermutationDecoder: + """This is a much harder problem than the shift decoder. There are 26! + permutations, so we can't try them all. Instead we have to search. + We want to search well, but there are many things to consider: + Unigram probabilities (E is the most common letter); Bigram probabilities + (TH is the most common bigram); word probabilities (I and A are the most + common one-letter words, etc.); etc. + We could represent a search state as a permutation of the 26 letters, + and alter the solution through hill climbing. With an initial guess + based on unigram probabilities, this would probably fare well. However, + I chose instead to have an incremental representation. A state is + represented as a letter-to-letter map; for example {'z': 'e'} to + represent that 'z' will be translated to 'e'.""" + + def __init__(self, training_text, ciphertext=None): + self.Pwords = UnigramWordModel(words(training_text)) + self.P1 = UnigramWordModel(training_text) # By letter + self.P2 = NgramWordModel(2, words(training_text)) # By letter pair + + def decode(self, ciphertext): + """Search for a decoding of the ciphertext.""" + self.ciphertext = canonicalize(ciphertext) + # reduce domain to speed up search + self.chardomain = {c for c in self.ciphertext if c != ' '} + problem = PermutationDecoderProblem(decoder=self) + solution = search.best_first_graph_search( + problem, lambda node: self.score(node.state)) + + solution.state[' '] = ' ' + return translate(self.ciphertext, lambda c: solution.state[c]) + + def score(self, code): + """Score is product of word scores, unigram scores, and bigram scores. + This can get very small, so we use logs and exp.""" + + # remake code dictionary to contain translation for all characters + full_code = code.copy() + full_code.update({x: x for x in self.chardomain if x not in code}) + full_code[' '] = ' ' + text = translate(self.ciphertext, lambda c: full_code[c]) + + # add small positive value to prevent computing log(0) + # TODO: Modify the values to make score more accurate + logP = (sum(np.log(self.Pwords[word] + 1e-20) for word in words(text)) + + sum(np.log(self.P1[c] + 1e-5) for c in text) + + sum(np.log(self.P2[b] + 1e-10) for b in bigrams(text))) + return -np.exp(logP) + + +class PermutationDecoderProblem(search.Problem): + """A search problem for breaking a permutation (substitution) cipher. + + A state is a partial mapping from plaintext characters to cipher characters; + the search incrementally assigns the most frequent unassigned character until + every character in the decoder's domain has been mapped.""" + + def __init__(self, initial=None, goal=None, decoder=None): + super().__init__(initial or hashabledict(), goal) + self.decoder = decoder + + def actions(self, state): + """Yield (plain_char, cipher_char) assignments extending ``state``: pick the + highest-probability unassigned plaintext char and pair it with each unused + cipher char.""" + search_list = [c for c in self.decoder.chardomain if c not in state] + target_list = [c for c in alphabet if c not in state.values()] + # Find the best character to replace + plain_char = max(search_list, key=lambda c: self.decoder.P1[c]) + for cipher_char in target_list: + yield (plain_char, cipher_char) + + def result(self, state, action): + """Return a new state extending ``state`` with the (plain, cipher) mapping + given by ``action``.""" + new_state = hashabledict(state) # copy to prevent hash issues + new_state[action[0]] = action[1] + return new_state + + def goal_test(self, state): + """We're done when all letters in search domain are assigned.""" + return len(state) >= len(self.decoder.chardomain) diff --git a/q-learning-9b/aima/utils.py b/q-learning-9b/aima/utils.py new file mode 100644 index 0000000..b3aaf1c --- /dev/null +++ b/q-learning-9b/aima/utils.py @@ -0,0 +1,872 @@ +"""Provides some utilities widely used by other modules""" + +from __future__ import annotations + +import bisect +import collections +import collections.abc +import functools +import heapq +import operator +import os.path +import random +from itertools import chain, combinations +from statistics import mean + +import numpy as np + + +# ______________________________________________________________________________ +# Functions on Sequences and Iterables + + +def sequence(iterable): + """Converts iterable to sequence, if it is not already one.""" + return iterable if isinstance(iterable, collections.abc.Sequence) else tuple([iterable]) + + +def remove_all(item, seq): + """Return a copy of seq (or string) with all occurrences of item removed.""" + if isinstance(seq, str): + return seq.replace(item, '') + elif isinstance(seq, set): + rest = seq.copy() + rest.remove(item) + return rest + else: + return [x for x in seq if x != item] + + +def unique(seq): + """Remove duplicate elements from seq. Assumes hashable elements.""" + return list(set(seq)) + + +def count(seq): + """Count the number of items in sequence that are interpreted as true.""" + return sum(map(bool, seq)) + + +def multimap(items): + """Given (key, val) pairs, return {key: [val, ....], ...}.""" + result = collections.defaultdict(list) + for (key, val) in items: + result[key].append(val) + return dict(result) + + +def multimap_items(mmap): + """Yield all (key, val) pairs stored in the multimap.""" + for (key, vals) in mmap.items(): + for val in vals: + yield key, val + + +def product(numbers): + """Return the product of the numbers, e.g. product([2, 3, 10]) == 60""" + result = 1 + for x in numbers: + result *= x + return result + + +def first(iterable, default=None): + """Return the first element of an iterable; or default.""" + return next(iter(iterable), default) + + +def is_in(elt, seq): + """Similar to (elt in seq), but compares with 'is', not '=='.""" + return any(x is elt for x in seq) + + +def mode(data): + """Return the most common data item. If there are ties, return any one of them.""" + [(item, count)] = collections.Counter(data).most_common(1) + return item + + +def power_set(iterable): + """power_set([1,2,3]) --> (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)""" + s = list(iterable) + return list(chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)))[1:] + + +def extend(s, var, val): + """Copy dict s and extend it by setting var to val; return copy.""" + return {**s, var: val} + + +def flatten(seqs): + """Flatten a sequence of sequences into a single flat list.""" + return sum(seqs, []) + + +# ______________________________________________________________________________ +# argmin and argmax + +identity = lambda x: x + + +def argmin_random_tie(seq, key=identity): + """Return a minimum element of seq; break ties at random.""" + return min(shuffled(seq), key=key) + + +def argmax_random_tie(seq, key=identity): + """Return an element with highest fn(seq[i]) score; break ties at random.""" + return max(shuffled(seq), key=key) + + +def shuffled(iterable): + """Randomly shuffle a copy of iterable.""" + items = list(iterable) + random.shuffle(items) + return items + + +# ______________________________________________________________________________ +# Statistical and mathematical functions + + +def histogram(values, mode=0, bin_function=None): + """Return a list of (value, count) pairs, summarizing the input values. + Sorted by increasing value, or if mode=1, by decreasing count. + If bin_function is given, map it over values first.""" + if bin_function: + values = map(bin_function, values) + + bins = {} + for val in values: + bins[val] = bins.get(val, 0) + 1 + + if mode: + return sorted(list(bins.items()), key=lambda x: (x[1], x[0]), reverse=True) + else: + return sorted(bins.items()) + + +def dot_product(x, y): + """Return the sum of the element-wise product of vectors x and y.""" + return sum(_x * _y for _x, _y in zip(x, y)) + + +def element_wise_product(x, y): + """Return vector as an element-wise product of vectors x and y.""" + assert len(x) == len(y) + return np.multiply(x, y) + + +def matrix_multiplication(x, *y): + r"""Return a matrix as a matrix-multiplication of x and arbitrary number of matrices \*y.""" + + result = x + for _y in y: + result = np.matmul(result, _y) + + return result + + +def vector_add(a, b): + """Component-wise addition of two vectors.""" + return tuple(map(operator.add, a, b)) + + +def scalar_vector_product(x, y): + """Return vector as a product of a scalar and a vector""" + return np.multiply(x, y) + + +def probability(p: float) -> bool: + """Return true with probability p.""" + return p > random.uniform(0.0, 1.0) + + +def weighted_sample_with_replacement(n, seq, weights): + """Pick n samples from seq at random, with replacement, with the + probability of each element in proportion to its corresponding + weight.""" + sample = weighted_sampler(seq, weights) + return [sample() for _ in range(n)] + + +def weighted_sampler(seq, weights): + """Return a random-sample function that picks from seq weighted by weights.""" + totals = [] + for w in weights: + totals.append(w + totals[-1] if totals else w) + return lambda: seq[bisect.bisect(totals, random.uniform(0, totals[-1]))] + + +def weighted_choice(choices): + """A weighted version of random.choice""" + # NOTE: should be replaced by random.choices if we port to Python 3.6 + + total = sum(w for _, w in choices) + r = random.uniform(0, total) + upto = 0 + for c, w in choices: + if upto + w >= r: + return c, w + upto += w + + +def rounder(numbers, d=4): + """Round a single number, or sequence of numbers, to d decimal places.""" + if isinstance(numbers, (int, float)): + return round(numbers, d) + else: + constructor = type(numbers) # Can be list, set, tuple, etc. + return constructor(rounder(n, d) for n in numbers) + + +def num_or_str(x: str) -> int | float | str: # TODO: rename as `atom` + """The argument is a string; convert to a number if possible, or strip it.""" + try: + return int(x) + except ValueError: + try: + return float(x) + except ValueError: + return str(x).strip() + + +def euclidean_distance(x, y) -> float: + """Return the Euclidean (L2) distance between vectors x and y.""" + return np.sqrt(sum((_x - _y) ** 2 for _x, _y in zip(x, y))) + + +def manhattan_distance(x, y) -> float: + """Return the Manhattan (L1) distance between vectors x and y.""" + return sum(abs(_x - _y) for _x, _y in zip(x, y)) + + +def hamming_distance(x, y) -> int: + """Return the number of positions at which vectors x and y differ.""" + return sum(_x != _y for _x, _y in zip(x, y)) + + +def cross_entropy_loss(x, y) -> float: + """Return the mean binary cross-entropy loss between targets x and predictions y.""" + return (-1.0 / len(x)) * sum(_x * np.log(_y) + (1 - _x) * np.log(1 - _y) for _x, _y in zip(x, y)) + + +def mean_squared_error_loss(x, y) -> float: + """Return the mean squared error loss between vectors x and y.""" + return (1.0 / len(x)) * sum((_x - _y) ** 2 for _x, _y in zip(x, y)) + + +def rms_error(x, y) -> float: + """Return the root-mean-square error between vectors x and y.""" + return np.sqrt(ms_error(x, y)) + + +def ms_error(x, y) -> float: + """Return the mean of the squared differences between vectors x and y.""" + return mean((_x - _y) ** 2 for _x, _y in zip(x, y)) + + +def mean_error(x, y) -> float: + """Return the mean of the absolute differences between vectors x and y.""" + return mean(abs(_x - _y) for _x, _y in zip(x, y)) + + +def mean_boolean_error(x, y) -> float: + """Return the fraction of positions at which vectors x and y differ.""" + return mean(_x != _y for _x, _y in zip(x, y)) + + +def normalize(dist): + """Multiply each number by a constant such that the sum is 1.0""" + if isinstance(dist, dict): + total = sum(dist.values()) + for key in dist: + dist[key] = dist[key] / total + assert 0 <= dist[key] <= 1 # probabilities must be between 0 and 1 + return dist + total = sum(dist) + return [(n / total) for n in dist] + + +def random_weights(min_value: float, max_value: float, num_weights: int) -> list: + """Return a list of num_weights random floats drawn uniformly from [min_value, max_value].""" + return [random.uniform(min_value, max_value) for _ in range(num_weights)] + + +def sigmoid(x): + """Return activation value of x with sigmoid function.""" + return 1 / (1 + np.exp(-x)) + + +def sigmoid_derivative(value): + """Return the derivative of the sigmoid, given its already-computed output value.""" + return value * (1 - value) + + +def elu(x: float, alpha: float = 0.01) -> float: + """Return the Exponential Linear Unit activation of x.""" + return x if x > 0 else alpha * (np.exp(x) - 1) + + +def elu_derivative(value: float, alpha: float = 0.01) -> float: + """Return the derivative of the ELU activation, given its output value.""" + return 1 if value > 0 else alpha * np.exp(value) + + +def tanh(x): + """Return the hyperbolic tangent activation of x.""" + return np.tanh(x) + + +def tanh_derivative(value): + """Return the derivative of tanh, given its already-computed output value.""" + return 1 - (value ** 2) + + +def leaky_relu(x: float, alpha: float = 0.01) -> float: + """Return the Leaky ReLU activation of x (slope alpha for negative inputs).""" + return x if x > 0 else alpha * x + + +def leaky_relu_derivative(value: float, alpha: float = 0.01) -> float: + """Return the derivative of the Leaky ReLU activation, given its output value.""" + return 1 if value > 0 else alpha + + +def relu(x: float) -> float: + """Return the Rectified Linear Unit activation of x, i.e. max(0, x).""" + return max(0, x) + + +def relu_derivative(value: float) -> int: + """Return the derivative of the ReLU activation, given its output value.""" + return 1 if value > 0 else 0 + + +def step(x: float) -> int: + """Return activation value of x with sign function""" + return 1 if x >= 0 else 0 + + +def gaussian(mean: float, st_dev: float, x: float) -> float: + """Given the mean and standard deviation of a distribution, it returns the probability of x.""" + return 1 / (np.sqrt(2 * np.pi) * st_dev) * np.e ** (-0.5 * (float(x - mean) / st_dev) ** 2) + + +def linear_kernel(x, y=None): + """Return the linear kernel (dot product) between x and y; defaults y to x.""" + if y is None: + y = x + return np.dot(x, y.T) + + +def polynomial_kernel(x, y=None, degree=2.0): + """Return the polynomial kernel (1 + x.y)**degree between x and y; defaults y to x.""" + if y is None: + y = x + return (1.0 + np.dot(x, y.T)) ** degree + + +def rbf_kernel(x, y=None, gamma=None): + """Radial-basis function kernel (aka squared-exponential kernel).""" + if y is None: + y = x + if gamma is None: + gamma = 1.0 / x.shape[1] # 1.0 / n_features + return np.exp(-gamma * (-2.0 * np.dot(x, y.T) + + np.sum(x * x, axis=1).reshape((-1, 1)) + np.sum(y * y, axis=1).reshape((1, -1)))) + + +# ______________________________________________________________________________ +# Grid Functions + + +orientations = EAST, NORTH, WEST, SOUTH = [(1, 0), (0, 1), (-1, 0), (0, -1)] +turns = LEFT, RIGHT = (+1, -1) + + +def turn_heading(heading, inc, headings=orientations): + """Return the heading reached by turning inc steps around the list of headings.""" + return headings[(headings.index(heading) + inc) % len(headings)] + + +def turn_right(heading): + """Return the heading obtained by turning right (clockwise) from heading.""" + return turn_heading(heading, RIGHT) + + +def turn_left(heading): + """Return the heading obtained by turning left (counter-clockwise) from heading.""" + return turn_heading(heading, LEFT) + + +def distance(a, b): + """The distance between two (x, y) points.""" + xA, yA = a + xB, yB = b + return np.hypot((xA - xB), (yA - yB)) + + +def distance_squared(a, b): + """The square of the distance between two (x, y) points.""" + xA, yA = a + xB, yB = b + return (xA - xB) ** 2 + (yA - yB) ** 2 + + +# ______________________________________________________________________________ +# Misc Functions + +class injection: + """Dependency injection of temporary values for global functions/classes/etc. + E.g., `with injection(DataBase=MockDataBase): ...`""" + + def __init__(self, **kwds): + self.new = kwds + + def __enter__(self): + self.old = {v: globals()[v] for v in self.new} + globals().update(self.new) + + def __exit__(self, type, value, traceback): + globals().update(self.old) + + +def memoize(fn, slot=None, maxsize=32): + """Memoize fn: make it remember the computed value for any argument list. + If slot is specified, store result in that slot of first argument. + If slot is false, use lru_cache for caching the values.""" + if slot: + def memoized_fn(obj, *args): + if hasattr(obj, slot): + return getattr(obj, slot) + else: + val = fn(obj, *args) + setattr(obj, slot, val) + return val + else: + @functools.lru_cache(maxsize=maxsize) + def memoized_fn(*args): + return fn(*args) + + return memoized_fn + + +def name(obj): + """Try to find some reasonable name for the object.""" + return (getattr(obj, 'name', 0) or getattr(obj, '__name__', 0) or + getattr(getattr(obj, '__class__', 0), '__name__', 0) or + str(obj)) + + +def isnumber(x): + """Is x a number?""" + return hasattr(x, '__int__') + + +def issequence(x): + """Is x a sequence?""" + return isinstance(x, collections.abc.Sequence) + + +def print_table(table, header=None, sep=' ', numfmt='{}'): + """Print a list of lists as a table, so that columns line up nicely. + header, if specified, will be printed as the first row. + numfmt is the format for all numbers; you might want e.g. '{:.2f}'. + (If you want different formats in different columns, + don't use print_table.) sep is the separator between columns.""" + justs = ['rjust' if isnumber(x) else 'ljust' for x in table[0]] + + if header: + table.insert(0, header) + + table = [[numfmt.format(x) if isnumber(x) else x for x in row] + for row in table] + + sizes = list(map(lambda seq: max(map(len, seq)), list(zip(*[map(str, row) for row in table])))) + + for row in table: + print(sep.join(getattr(str(x), j)(size) for (j, size, x) in zip(justs, sizes, row))) + + +def open_data(name, mode='r'): + """Open and return the file named name from the aima-data directory.""" + aima_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + aima_file = os.path.join(aima_root, *['aima-data', name]) + + return open(aima_file, mode=mode) + + +def failure_test(algorithm, tests): + """Grades the given algorithm based on how many tests it passes. + Most algorithms have arbitrary output on correct execution, which is difficult + to check for correctness. On the other hand, a lot of algorithms output something + particular on fail (for example, False, or None). + tests is a list with each element in the form: (values, failure_output).""" + return mean(int(algorithm(x) != y) for x, y in tests) + + +# ______________________________________________________________________________ +# Expressions + +# See https://docs.python.org/3/reference/expressions.html#operator-precedence +# See https://docs.python.org/3/reference/datamodel.html#special-method-names + +class Expr: + """A mathematical expression with an operator and 0 or more arguments. + op is a str like '+' or 'sin'; args are Expressions. + Expr('x') or Symbol('x') creates a symbol (a nullary Expr). + Expr('-', x) creates a unary; Expr('+', x, 1) creates a binary.""" + + def __init__(self, op, *args): + self.op = str(op) + self.args = args + + # Operator overloads + def __neg__(self): + return Expr('-', self) + + def __pos__(self): + return Expr('+', self) + + def __invert__(self): + return Expr('~', self) + + def __add__(self, rhs): + return Expr('+', self, rhs) + + def __sub__(self, rhs): + return Expr('-', self, rhs) + + def __mul__(self, rhs): + return Expr('*', self, rhs) + + def __pow__(self, rhs): + return Expr('**', self, rhs) + + def __mod__(self, rhs): + return Expr('%', self, rhs) + + def __and__(self, rhs): + return Expr('&', self, rhs) + + def __xor__(self, rhs): + return Expr('^', self, rhs) + + def __rshift__(self, rhs): + return Expr('>>', self, rhs) + + def __lshift__(self, rhs): + return Expr('<<', self, rhs) + + def __truediv__(self, rhs): + return Expr('/', self, rhs) + + def __floordiv__(self, rhs): + return Expr('//', self, rhs) + + def __matmul__(self, rhs): + return Expr('@', self, rhs) + + def __or__(self, rhs): + """Allow both P | Q, and P |'==>'| Q.""" + if isinstance(rhs, Expression): + return Expr('|', self, rhs) + else: + return PartialExpr(rhs, self) + + # Reverse operator overloads + def __radd__(self, lhs): + return Expr('+', lhs, self) + + def __rsub__(self, lhs): + return Expr('-', lhs, self) + + def __rmul__(self, lhs): + return Expr('*', lhs, self) + + def __rdiv__(self, lhs): + return Expr('/', lhs, self) + + def __rpow__(self, lhs): + return Expr('**', lhs, self) + + def __rmod__(self, lhs): + return Expr('%', lhs, self) + + def __rand__(self, lhs): + return Expr('&', lhs, self) + + def __rxor__(self, lhs): + return Expr('^', lhs, self) + + def __ror__(self, lhs): + return Expr('|', lhs, self) + + def __rrshift__(self, lhs): + return Expr('>>', lhs, self) + + def __rlshift__(self, lhs): + return Expr('<<', lhs, self) + + def __rtruediv__(self, lhs): + return Expr('/', lhs, self) + + def __rfloordiv__(self, lhs): + return Expr('//', lhs, self) + + def __rmatmul__(self, lhs): + return Expr('@', lhs, self) + + def __call__(self, *args): + """Call: if 'f' is a Symbol, then f(0) == Expr('f', 0).""" + if self.args: + raise ValueError('Can only do a call for a Symbol, not an Expr') + else: + return Expr(self.op, *args) + + # Equality and repr + def __eq__(self, other): + """x == y' evaluates to True or False; does not build an Expr.""" + return isinstance(other, Expr) and self.op == other.op and self.args == other.args + + def __lt__(self, other): + return isinstance(other, Expr) and str(self) < str(other) + + def __hash__(self): + return hash(self.op) ^ hash(self.args) + + def __repr__(self): + op = self.op + args = [str(arg) for arg in self.args] + if op.isidentifier(): # f(x) or f(x, y) + return '{}({})'.format(op, ', '.join(args)) if args else op + elif len(args) == 1: # -x or -(x + 1) + return op + args[0] + else: # (x - y) + opp = (' ' + op + ' ') + return '(' + opp.join(args) + ')' + + +# An 'Expression' is either an Expr or a Number. +# Symbol is not an explicit type; it is any Expr with 0 args. + + +Number = (int, float, complex) +Expression = (Expr, Number) + + +def Symbol(name): + """A Symbol is just an Expr with no args.""" + return Expr(name) + + +def symbols(names): + """Return a tuple of Symbols; names is a comma/whitespace delimited str.""" + return tuple(Symbol(name) for name in names.replace(',', ' ').split()) + + +def subexpressions(x): + """Yield the subexpressions of an Expression (including x itself).""" + yield x + if isinstance(x, Expr): + for arg in x.args: + yield from subexpressions(arg) + + +def arity(expression): + """The number of sub-expressions in this expression.""" + if isinstance(expression, Expr): + return len(expression.args) + else: # expression is a number + return 0 + + +# For operators that are not defined in Python, we allow new InfixOps: + + +class PartialExpr: + r"""Given 'P \|'==>'\| Q, first form PartialExpr('==>', P), then combine with Q.""" + + def __init__(self, op, lhs): + self.op, self.lhs = op, lhs + + def __or__(self, rhs): + return Expr(self.op, self.lhs, rhs) + + def __repr__(self): + return "PartialExpr('{}', {})".format(self.op, self.lhs) + + +def expr(x): + r"""Shortcut to create an Expression. x is a str in which: + - identifiers are automatically defined as Symbols. + - ==> is treated as an infix \|'==>'\|, as are <== and <=>. + If x is already an Expression, it is returned unchanged. Example: + + >>> expr('P & Q ==> Q') + ((P & Q) ==> Q) + """ + return eval(expr_handle_infix_ops(x), defaultkeydict(Symbol)) if isinstance(x, str) else x + + +infix_ops = '==> <== <=>'.split() + + +def expr_handle_infix_ops(x): + r"""Given a str, return a new str with ==> replaced by \|'==>'\|, etc. + + >>> expr_handle_infix_ops('P ==> Q') + "P |'==>'| Q" + """ + for op in infix_ops: + x = x.replace(op, '|' + repr(op) + '|') + return x + + +class defaultkeydict(collections.defaultdict): + """Like defaultdict, but the default_factory is a function of the key. + >>> d = defaultkeydict(len); d['four'] + 4 + """ + + def __missing__(self, key): + self[key] = result = self.default_factory(key) + return result + + +class hashabledict(dict): + """Allows hashing by representing a dictionary as tuple of key:value pairs. + May cause problems as the hash value may change during runtime.""" + + def __hash__(self): + return 1 + + +# ______________________________________________________________________________ +# Queues: Stack, FIFOQueue, PriorityQueue +# Stack and FIFOQueue are implemented as list and collection.deque +# PriorityQueue is implemented here + + +class PriorityQueue: + """A Queue in which the minimum (or maximum) element (as determined by f and + order) is returned first. + If order is 'min', the item with minimum f(x) is + returned first; if order is 'max', then it is the item with maximum f(x). + Also supports dict-like lookup.""" + + def __init__(self, order='min', f=lambda x: x): + self.heap = [] + # monotonic tie-breaker so items with equal priority are never compared + # (which would fail for non-comparable items); ties keep insertion order + self.counter = 0 + if order == 'min': + self.f = f + elif order == 'max': # now item with max f(x) + self.f = lambda x: -f(x) # will be popped first + else: + raise ValueError("Order must be either 'min' or 'max'.") + + def append(self, item): + """Insert item at its correct position.""" + heapq.heappush(self.heap, (self.f(item), self.counter, item)) + self.counter += 1 + + def extend(self, items): + """Insert each item in items at its correct position.""" + for item in items: + self.append(item) + + def pop(self): + """Pop and return the item (with min or max f(x) value) + depending on the order.""" + if self.heap: + return heapq.heappop(self.heap)[-1] + else: + raise Exception('Trying to pop from empty PriorityQueue.') + + def __len__(self): + """Return current capacity of PriorityQueue.""" + return len(self.heap) + + def __contains__(self, key): + """Return True if the key is in PriorityQueue.""" + return any(item == key for *_, item in self.heap) + + def __getitem__(self, key): + """Returns the first value associated with key in PriorityQueue. + Raises KeyError if key is not present.""" + for value, _, item in self.heap: + if item == key: + return value + raise KeyError(str(key) + " is not in the priority queue") + + def __delitem__(self, key): + """Delete the first occurrence of key.""" + try: + del self.heap[[item == key for *_, item in self.heap].index(True)] + except ValueError: + raise KeyError(str(key) + " is not in the priority queue") + heapq.heapify(self.heap) + + +# ______________________________________________________________________________ +# Useful Shorthands + + +class Bool(int): + """Just like `bool`, except values display as 'T' and 'F' instead of 'True' and 'False'.""" + __str__ = __repr__ = lambda self: 'T' if self else 'F' + + +T = Bool(True) +F = Bool(False) + + +# ______________________________________________________________________________ +# 4th-edition additions (kernels, convolution, Monte Carlo tree search helpers) + +def gaussian_kernel(size=3): + """Return a length-size 1D Gaussian kernel centred at the middle (fixed st_dev 0.1).""" + return [gaussian((size - 1) / 2, 0.1, x) for x in range(size)] + + +def gaussian_kernel_1D(size=3, sigma=0.5): + """Return a length-size 1D Gaussian kernel centred at the middle with st_dev sigma.""" + return [gaussian((size - 1) / 2, sigma, x) for x in range(size)] + + +def gaussian_kernel_2D(size=3, sigma=0.5): + """Return a size x size 2D Gaussian kernel with st_dev sigma, normalized to sum to 1.""" + x, y = np.mgrid[-size // 2 + 1:size // 2 + 1, -size // 2 + 1:size // 2 + 1] + g = np.exp(-((x ** 2 + y ** 2) / (2.0 * sigma ** 2))) + return g / g.sum() + + +def conv1D(x, k): + """1D convolution. x: input vector; K: kernel vector.""" + return np.convolve(x, k, mode='same') + + +def map_vector(f, x): + """Apply function f to iterable x.""" + return [map_vector(f, _x) for _x in x] if hasattr(x, '__iter__') else list(map(f, [x]))[0] + + +class MCT_Node: + """Node in the Monte Carlo search tree, keeps track of the children states.""" + + def __init__(self, parent=None, state=None, U=0, N=0): + self.__dict__.update(parent=parent, state=state, U=U, N=N) + self.children = {} + self.actions = None + + +def ucb(n, C=1.4): + """Return the UCB1 score of node n (exploitation plus C-weighted exploration term). + + Unvisited nodes (n.N == 0) score infinity so they are selected first; used to guide + selection in Monte Carlo tree search. + """ + return np.inf if n.N == 0 else n.U / n.N + C * np.sqrt(np.log(n.parent.N) / n.N) diff --git a/q-learning-9b/context/environment.md b/q-learning-9b/context/environment.md new file mode 100644 index 0000000..bd29c1c --- /dev/null +++ b/q-learning-9b/context/environment.md @@ -0,0 +1,65 @@ +# Vacuum Robot Environment + +## Overview + +The vacuum robot operates in a 4x4 grid environment. + +Lab 5 uses the grid provided in Lab 4, with the movement and reward rules agreed by the group. + +## Grid Layout + +| start | 0 | 0 | 0 | +|-------|----|---|----| +| 0 | X | 0 | 0 | +| 0 | 0 | 0 | 0 | +| 0 | -1 | 0 | 10 | + +## Coordinate System + +For documentation purposes, positions use zero-based (row, column) coordinates. + +The top-left cell is (0, 0). + +## Important Locations + +| Position | Value | Description | +|----------|-------|-------------| +| (0, 0) | start | Robot's starting position | +| (1, 1) | X | Obstacle | +| (3, 1) | -1 | Negative reward cell | +| (3, 3) | 10 | Positive reward cell | +| Other | 0 | Normal cells | + +## Environment Characteristics + +The environment contains: + +- 16 total grid positions +- One obstacle +- One starting position +- One negative reward cell +- One positive reward cell + +The robot will use Q-learning to learn how to interact with this environment. + +## Movement + +- Actions: Up, Down, Left, and Right. +- Move in the chosen direction with probability 80%. +- Move in either perpendicular direction with probability 10% each. +- If the resulting move hits the obstacle or a grid boundary, stay in place. + +## Rewards and Episodes + +- Start each episode at (0, 0). +- Ordinary moves receive the living reward, initially proposed as -0.04. +- Blocked moves also receive the living reward. +- Entering (3, 1) gives -1 and ends the episode. +- Entering (3, 3) gives +10 and ends the episode. +- Do not add the living reward to a terminal reward. + +## Q-learning + +The simulator uses these movement and reward rules. The learning agent observes its current position, takes an action, and receives the reward, next position, and episode status. It does not read explicit T and R tables or the environment's reward map. + +The environment should include a dedicated function to find the recharge location, addressing the Lab 3 feedback. This function belongs to the environment and does not provide a route to the learning agent. diff --git a/q-learning-9b/context/requirements.md b/q-learning-9b/context/requirements.md new file mode 100644 index 0000000..37e716c --- /dev/null +++ b/q-learning-9b/context/requirements.md @@ -0,0 +1,96 @@ +# CS440 Lab 5 Requirements + +## Assignment + +Lab 5: Solve Lab 4 Using Q-learning + +## Objective + +Implement the Q-learning vacuum robot using the Lab 4 environment. + +The assignment requires three of these four code features: hooks, MCP, skills, and subagents. + +This repository will use: + +1. Subagents (building on Isolate) +2. Hooks +3. Skills + +Write is not a separate demonstration in this repository. + +## Environment + +The vacuum robot uses the following 4x4 grid: + +| start | 0 | 0 | 0 | +|-------|----|---|----| +| 0 | X | 0 | 0 | +| 0 | 0 | 0 | 0 | +| 0 | -1 | 0 | 10 | + +The learning agent does not receive explicit T and R tables. It learns from observed transitions and rewards. + +Use the agreed 80/10/10 movement and reward rules in `environment.md`. + +## Current Lab Scope + +Focus on: +- Implementing and evaluating the Q-learning robot +- Using and demonstrating subagents, hooks, and skills +- Selecting and explaining the hyperparameters +- Showing the running solution and documenting how to run it + +## Proposed Hyperparameters + +Starting values to test: + +- Living reward: -0.04 on nonterminal steps, encouraging shorter paths. +- Discount factor: 0.95, giving future rewards substantial value. +- Learning rate: 0.1, updating action values gradually. +- Epsilon: 0.2 during training, exploring on 20% of action selections. +- Evaluation epsilon: 0, following the learned policy while movement remains stochastic. + +These are proposed starting values, not verified optimal settings. Report the actual values used and measured results. + +## Required Submission + +The group must submit the following on Canvas: + +### 1. Repository Links + +A text file containing the link to the repository configured with the selected code features. + +### 2. README + +A README file documenting the project structure, setup, and how to run the solution. + +### 3. Video Presentation + +A video showing the running solution and explaining the code features and hyperparameters. + +Requirements: +- Maximum duration: 3 minutes +- Every group member must present for a fair amount of time +- Students must show their faces while presenting +- Use a professional background +- Explain the code feature decisions +- Explain the hyperparameter decisions +- Show the running solution + +The video must be private and shared only with the instructor team. + +If submitting a video link, put it in group_name.txt for a group submission or firstName_lastName.txt for an individual submission. + +A private video link or a video file such as MP4 may be submitted. The link must work for the instructor team and must not allow other students access. + +Submissions without a video receive zero points. The instructor team may ask any participant questions about the implementation. + +## AI Policy + +The use of AI is allowed for this assignment. + +## Strategy 4 Objective + +Demonstrate how Claude Code delegates focused tasks to specialists, each receiving only the context needed for its role. + +The demonstration should show separate specialist calls, relevant source files, and short summaries returned to the main agent. \ No newline at end of file diff --git a/q-learning-9b/learning_todos.py b/q-learning-9b/learning_todos.py new file mode 100644 index 0000000..497fbc7 --- /dev/null +++ b/q-learning-9b/learning_todos.py @@ -0,0 +1,135 @@ +def choose_action(q_values, state, epsilon, rng): + """Epsilon-greedy: random action with probability epsilon, else the best one (ties broken randomly).""" + if state not in q_values or rng.random() < epsilon: + # Explore: pick random action from ACTIONS + return rng.choice(ACTIONS) + else: + # Exploit: find best Q-value(s) + best_q = max(q_values[state].values()) + best_actions = [a for a, q in q_values[state].items() if q == best_q] + return rng.choice(best_actions) + + +def outcome_of(reward, done): + """Label how an episode ended, using only the observed reward.""" + if done: + return "goal" if reward > 0 else "trap" + return "timeout" + + +def q_learning(env, episodes, max_steps, alpha=ALPHA, gamma=GAMMA, + epsilon=EPSILON, seed=0): + """Learn action values from environment interactions. Returns (q_values, results).""" + random.seed(seed) + rng = random.Random(seed) + q_values = {} + results = [] + robot = Agent(lambda percept: None) + + for episode in range(1, episodes + 1): + state = env.reset(robot) + if state not in q_values: + q_values[state] = {a: 0.0 for a in ACTIONS} + + total_reward = 0 + steps = 0 + done = False + + while not done and steps < max_steps: + action = choose_action(q_values, state, epsilon, rng) + next_state, reward, done = env.execute_action(robot, action) + + if state not in q_values: + q_values[state] = {a: 0.0 for a in ACTIONS} + + old_score = q_values[state][action] + + if done: + target = reward + else: + if next_state not in q_values: + q_values[next_state] = {a: 0.0 for a in ACTIONS} + target = reward + gamma * max(q_values[next_state].values()) + + q_values[state][action] = old_score + alpha * (target - old_score) + + total_reward += reward + state = next_state + steps += 1 + + results.append({ + "episode": episode, + "steps": steps, + "reward": total_reward, + "outcome": outcome_of(reward, done) + }) + + return (q_values, results) + + +def derive_policy(q_values): + """Best action for each position the robot visited.""" + policy = {} + for state, actions in q_values.items(): + best_action = max(actions, key=actions.get) + policy[state] = best_action + return policy + + +def evaluate_agent(env, q_values, episodes, max_steps, seed=1, show_paths=3): + """Evaluate the learned policy without changing Q-values (epsilon = EVALUATION_EPSILON).""" + random.seed(seed) + rng = random.Random(seed) + robot = Agent(lambda percept: None) + + total_reward = 0 + total_steps = 0 + goal_count = 0 + trap_count = 0 + timeout_count = 0 + + for episode in range(episodes): + state = env.reset(robot) + done = False + last_reward = 0 + steps = 0 + path = [] + + while not done and steps < max_steps: + action = choose_action(q_values, state, EVALUATION_EPSILON, rng) + next_state, reward, done = env.execute_action(robot, action) + + if episode < show_paths: + path.append(f"{action} -> {next_state}") + + total_reward += reward + last_reward = reward + state = next_state + steps += 1 + + outcome = outcome_of(last_reward, done) + if outcome == "goal": + goal_count += 1 + elif outcome == "trap": + trap_count += 1 + else: + timeout_count += 1 + + total_steps += steps + + if episode < show_paths: + print(" | ".join(path)) + + average_reward = total_reward / episodes + average_steps = total_steps / episodes + success_rate = goal_count / episodes + + return { + "episodes": episodes, + "goal": goal_count, + "trap": trap_count, + "timeout": timeout_count, + "average_reward": average_reward, + "average_steps": average_steps, + "success_rate": success_rate + } diff --git a/q-learning-9b/q_learning.ipynb b/q-learning-9b/q_learning.ipynb new file mode 100644 index 0000000..09dd120 --- /dev/null +++ b/q-learning-9b/q_learning.ipynb @@ -0,0 +1,533 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "lab5-overview", + "metadata": {}, + "source": [ + "# CS440 Lab 5: Q-learning Vacuum Robot\n", + "\n", + "Adapted from the group's Lab 3 `markov.ipynb`. This notebook implements\n", + "the environment and training, evaluation, and recording setup for the demo.\n", + "The five learner functions are left as TODOs.\n", + "\n", + "Kept: the environment structure, 80/10/10 movement helpers, start lookup,\n", + "and a simplified policy display. The start is now top-left for Lab 4/5.\n", + "\n", + "Removed: value iteration, expected-utility calculations, the explicit\n", + "transition table used by the planner, old utilities, the policy runner,\n", + "and the three living-cost experiments and their saved outputs.\n", + "\n", + "The class name `MDPEnvironment` is retained from Lab 3; it represents the\n", + "simulator, not a value-iteration solver. Q-learning learns from observed\n", + "transitions and rewards without reading the simulator's transition probabilities or reward map.\n", + "\n", + "## Before running\n", + "\n", + "The original notebook used `%run bootstrap.ipynb` and `aima.agents`.\n", + "The bootstrap command was removed because that file was not supplied.\n", + "The course's AIMA package is now included in `aima/`.\n", + "Install dependencies with `python -m pip install -r requirements.txt` in\n", + "the notebook's Python environment. The local package supplies\n", + "`aima.agents` without the bootstrap notebook.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "# Uses the local course AIMA package; install requirements.txt first.\n", + "from aima.agents import *\n", + "import random\n" + ] + }, + { + "cell_type": "markdown", + "id": "environment-notes", + "metadata": {}, + "source": [ + "## Environment\n", + "\n", + "Movement helpers below are adapted from Lab 3. `execute_action` returns\n", + "the next position, reward, and episode status. `reset` starts a new episode.\n", + "\n", + "Agreed rules: 80% intended direction, 10% each perpendicular direction;\n", + "blocked moves stay in place. Ordinary and blocked moves receive the living\n", + "reward. Entering -1 or +10 ends the episode with exactly that reward.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "environment", + "metadata": {}, + "outputs": [], + "source": [ + "default_world = [['start', 0, 0, 0],\n", + " [0, 'X', 0, 0],\n", + " [0, 0, 0, 0],\n", + " [0, -1, 0, 10]]\n", + "\n", + "class MDPEnvironment(Environment):\n", + " def __init__(self, env=default_world, living_reward=-0.04):\n", + " super().__init__()\n", + " self.status = env\n", + " self.living_reward = living_reward\n", + " self.done = False\n", + "\n", + " def percept(self, agent):\n", + " \"\"\"Returns the agent's location, and the location status.\"\"\"\n", + " return agent.location, self.status[agent.location[0]][agent.location[1]]\n", + "\n", + " def reset(self, agent):\n", + " \"\"\"Start a new episode: put the agent on the start cell and clear its score.\"\"\"\n", + " agent.location = self.default_location(agent)\n", + " agent.performance = 0\n", + " agent.alive = True\n", + " self.done = False\n", + " return agent.location\n", + "\n", + " def execute_action(self, agent, action):\n", + " \"\"\"Return (next_location, reward, done) for one action.\"\"\"\n", + " if self.done:\n", + " raise RuntimeError(\"Episode is over; call reset(agent) first.\")\n", + "\n", + " if action == 'Right':\n", + " self._attempt(agent, self._right, self._up, self._down)\n", + " elif action == 'Up':\n", + " self._attempt(agent, self._up, self._right, self._left)\n", + " elif action == 'Down':\n", + " self._attempt(agent, self._down, self._left, self._right)\n", + " elif action == 'Left':\n", + " self._attempt(agent, self._left, self._up, self._down)\n", + " else:\n", + " raise ValueError(f\"Unknown action: {action}\")\n", + "\n", + " cell = self.status[agent.location[0]][agent.location[1]]\n", + " if cell in (10, -1):\n", + " reward = cell # exactly +10 or -1, no living reward added\n", + " self.done = True\n", + " agent.alive = (cell == 10)\n", + " else:\n", + " reward = self.living_reward # ordinary and blocked moves\n", + "\n", + " agent.performance += reward\n", + " return agent.location, reward, self.done\n", + " \n", + " def _attempt(self, agent, intended_action, accidental_action1, accidental_action2):\n", + " slip = random.random()\n", + " tup = slip < 0.8, slip >= 0.9\n", + "\n", + " match tup:\n", + " case True, _:\n", + " intended_action(agent)\n", + " case _, False:\n", + " accidental_action1(agent)\n", + " case _, True:\n", + " accidental_action2(agent)\n", + "\n", + " \"\"\" Movements are named so that they correspond to the actual direction.\n", + " For example, an agent at list indices (0, 1)\n", + " could go down to make the new list indices (1, 1),\n", + " or it could go left to make the new list indices (0, 0)\"\"\"\n", + "\n", + " def _down(self, agent):\n", + " newX = agent.location[0] + 1\n", + " Y = agent.location[1]\n", + " if not (newX >= len(self.status) or self.status[newX][Y] == 'X'):\n", + " agent.location = newX, Y\n", + "\n", + " def _up(self, agent):\n", + " newX = agent.location[0] - 1\n", + " Y = agent.location[1]\n", + " if not (newX < 0 or self.status[newX][Y] == 'X'):\n", + " agent.location = newX, Y\n", + "\n", + " def _right(self, agent):\n", + " X = agent.location[0]\n", + " newY = agent.location[1] + 1\n", + " if not (newY >= len(self.status[0]) or self.status[X][newY] == 'X'):\n", + " agent.location = X, newY\n", + "\n", + " def _left(self, agent):\n", + " X = agent.location[0]\n", + " newY = agent.location[1] - 1\n", + " if not (newY < 0 or self.status[X][newY] == 'X'):\n", + " agent.location = X, newY\n", + "\n", + " def default_location(self, thing):\n", + " for i, list in enumerate(self.status):\n", + " for j, element in enumerate(list):\n", + " if element == 'start':\n", + " return (i, j)\n", + "\n", + " def find_recharge_location(self):\n", + " \"\"\"Return the (row, col) of the cell labeled 10.\"\"\"\n", + " for i, row in enumerate(self.status):\n", + " for j, element in enumerate(row):\n", + " if element == 10:\n", + " return (i, j)\n", + " return None\n", + "\n", + " def __str__(self):\n", + " return str(self.status)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "parameters", + "metadata": {}, + "outputs": [], + "source": [ + "ACTIONS = ['Up', 'Down', 'Left', 'Right']\n", + "\n", + "LIVING_REWARD = -0.04\n", + "GAMMA = 0.95\n", + "ALPHA = 0.1\n", + "EPSILON = 0.2\n", + "EVALUATION_EPSILON = 0.0\n", + "\n", + "# Training and evaluation settings\n", + "TRAIN_EPISODES = 2000\n", + "EVAL_EPISODES = 500\n", + "MAX_STEPS = 100\n", + "TRAIN_SEED = 0\n", + "EVAL_SEED = 1" + ] + }, + { + "cell_type": "markdown", + "id": "learning-notes", + "metadata": {}, + "source": [ + "## Q-learning\n", + "\n", + "The learner must not read `env.status`, `_attempt`, or recharge-location\n", + "helpers to derive a policy. Only the simulator uses those internals.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "learning-todos", + "metadata": {}, + "outputs": [], + "source": [ + "def choose_action(q_values, state, epsilon, rng):\n", + " \"\"\"Epsilon-greedy: random action with probability epsilon, else the best one (ties broken randomly).\"\"\"\n", + " if state not in q_values or rng.random() < epsilon:\n", + " # Explore: pick random action from ACTIONS\n", + " return rng.choice(ACTIONS)\n", + " else:\n", + " # Exploit: find best Q-value(s)\n", + " best_q = max(q_values[state].values())\n", + " best_actions = [a for a, q in q_values[state].items() if q == best_q]\n", + " return rng.choice(best_actions)\n", + "\n", + "\n", + "def outcome_of(reward, done):\n", + " \"\"\"Label how an episode ended, using only the observed reward.\"\"\"\n", + " if done:\n", + " return \"goal\" if reward > 0 else \"trap\"\n", + " return \"timeout\"\n", + "\n", + "\n", + "def q_learning(env, episodes, max_steps, alpha=ALPHA, gamma=GAMMA,\n", + " epsilon=EPSILON, seed=0):\n", + " \"\"\"Learn action values from environment interactions. Returns (q_values, results).\"\"\"\n", + " random.seed(seed)\n", + " rng = random.Random(seed)\n", + " q_values = {}\n", + " results = []\n", + " robot = Agent(lambda percept: None)\n", + "\n", + " for episode in range(1, episodes + 1):\n", + " state = env.reset(robot)\n", + " if state not in q_values:\n", + " q_values[state] = {a: 0.0 for a in ACTIONS}\n", + " \n", + " total_reward = 0\n", + " steps = 0\n", + " done = False\n", + " \n", + " while not done and steps < max_steps:\n", + " action = choose_action(q_values, state, epsilon, rng)\n", + " next_state, reward, done = env.execute_action(robot, action)\n", + " \n", + " if state not in q_values:\n", + " q_values[state] = {a: 0.0 for a in ACTIONS}\n", + " \n", + " old_score = q_values[state][action]\n", + " \n", + " if done:\n", + " target = reward\n", + " else:\n", + " if next_state not in q_values:\n", + " q_values[next_state] = {a: 0.0 for a in ACTIONS}\n", + " target = reward + gamma * max(q_values[next_state].values())\n", + " \n", + " q_values[state][action] = old_score + alpha * (target - old_score)\n", + " \n", + " total_reward += reward\n", + " state = next_state\n", + " steps += 1\n", + " \n", + " results.append({\n", + " \"episode\": episode,\n", + " \"steps\": steps,\n", + " \"reward\": total_reward,\n", + " \"outcome\": outcome_of(reward, done)\n", + " })\n", + " \n", + " return (q_values, results)\n", + "\n", + "\n", + "def derive_policy(q_values):\n", + " \"\"\"Best action for each position the robot visited.\"\"\"\n", + " policy = {}\n", + " for state, actions in q_values.items():\n", + " best_action = max(actions, key=actions.get)\n", + " policy[state] = best_action\n", + " return policy\n", + "\n", + "\n", + "def evaluate_agent(env, q_values, episodes, max_steps, seed=1, show_paths=3):\n", + " \"\"\"Evaluate the learned policy without changing Q-values (epsilon = EVALUATION_EPSILON).\"\"\"\n", + " random.seed(seed)\n", + " rng = random.Random(seed)\n", + " robot = Agent(lambda percept: None)\n", + " \n", + " total_reward = 0\n", + " total_steps = 0\n", + " goal_count = 0\n", + " trap_count = 0\n", + " timeout_count = 0\n", + " \n", + " for episode in range(episodes):\n", + " state = env.reset(robot)\n", + " done = False\n", + " last_reward = 0\n", + " steps = 0\n", + " path = []\n", + " \n", + " while not done and steps < max_steps:\n", + " action = choose_action(q_values, state, EVALUATION_EPSILON, rng)\n", + " next_state, reward, done = env.execute_action(robot, action)\n", + " \n", + " if episode < show_paths:\n", + " path.append(f\"{action} -> {next_state}\")\n", + " \n", + " total_reward += reward\n", + " last_reward = reward\n", + " state = next_state\n", + " steps += 1\n", + " \n", + " outcome = outcome_of(last_reward, done)\n", + " if outcome == \"goal\":\n", + " goal_count += 1\n", + " elif outcome == \"trap\":\n", + " trap_count += 1\n", + " else:\n", + " timeout_count += 1\n", + " \n", + " total_steps += steps\n", + " \n", + " if episode < show_paths:\n", + " print(\" | \".join(path))\n", + " \n", + " average_reward = total_reward / episodes\n", + " average_steps = total_steps / episodes\n", + " success_rate = goal_count / episodes\n", + " \n", + " return {\n", + " \"episodes\": episodes,\n", + " \"goal\": goal_count,\n", + " \"trap\": trap_count,\n", + " \"timeout\": timeout_count,\n", + " \"average_reward\": average_reward,\n", + " \"average_steps\": average_steps,\n", + " \"success_rate\": success_rate\n", + " }\n" + ] + }, + { + "cell_type": "markdown", + "id": "display-notes", + "metadata": {}, + "source": [ + "## Policy display\n", + "\n", + "Call this helper after deriving a policy from the learned Q-table.\n", + "The default world is used here only to display obstacles and terminal cells.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "policy-display", + "metadata": {}, + "outputs": [], + "source": [ + "direction = {'Up': 'U', 'Down': 'D', 'Left': 'L', 'Right': 'R'}\n", + "def print_policy(policy, world=default_world): \n", + " print(\"Policy: \")\n", + " for i in range(len(world)):\n", + " row = []\n", + " for g in range(len(world[0])):\n", + " postion = world[i][g]\n", + " if postion == \"X\":\n", + " row.append(' X ')\n", + " elif postion == -1:\n", + " row.append('TRAP')\n", + " elif postion == 10:\n", + " row.append('RECHARGE')\n", + " else:\n", + " row.append(direction[policy[(i, g)]] if (i, g) in policy else '?')\n", + " print(' '.join(row))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cacc3ebd", + "metadata": {}, + "outputs": [], + "source": [ + "env = MDPEnvironment(living_reward=LIVING_REWARD)\n", + "print(\"Recharge location:\", env.find_recharge_location())\n", + "\n", + "q_values, training = q_learning(env, TRAIN_EPISODES, MAX_STEPS, alpha=ALPHA,\n", + " gamma=GAMMA, epsilon=EPSILON, seed=TRAIN_SEED)\n", + "\n", + "last = training[-100:]\n", + "print(\"Training, last 100 episodes:\",\n", + " {o: sum(r['outcome'] == o for r in last) for o in ('goal', 'trap', 'timeout')},\n", + " f\"average reward {sum(r['reward'] for r in last) / len(last):.2f}\")\n", + "\n", + "policy = derive_policy(q_values)\n", + "print_policy(policy)\n", + "print()\n", + "evaluation = evaluate_agent(env, q_values, EVAL_EPISODES, MAX_STEPS, seed=EVAL_SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "937278bc", + "metadata": {}, + "source": [ + "# Hyperparameter Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45565a49", + "metadata": {}, + "outputs": [], + "source": [ + "# Writes results.json so the README and presentation can quote real numbers.\n", + "import json\n", + "from datetime import datetime\n", + "\n", + "def window_summary(rows):\n", + " return {'episodes': len(rows),\n", + " **{o: sum(r['outcome'] == o for r in rows) for o in ('goal', 'trap', 'timeout')},\n", + " 'average_reward': sum(r['reward'] for r in rows) / len(rows),\n", + " 'average_steps': sum(r['steps'] for r in rows) / len(rows)}\n", + "\n", + "record = {\n", + " 'recorded_at': datetime.now().isoformat(timespec='seconds'),\n", + " 'hyperparameters': {'LIVING_REWARD': LIVING_REWARD, 'GAMMA': GAMMA, 'ALPHA': ALPHA,\n", + " 'EPSILON': EPSILON, 'EVALUATION_EPSILON': EVALUATION_EPSILON,\n", + " 'TRAIN_EPISODES': TRAIN_EPISODES, 'EVAL_EPISODES': EVAL_EPISODES,\n", + " 'MAX_STEPS': MAX_STEPS, 'TRAIN_SEED': TRAIN_SEED, 'EVAL_SEED': EVAL_SEED},\n", + " 'training_first_100': window_summary(training[:100]),\n", + " 'training_last_100': window_summary(training[-100:]),\n", + " 'evaluation': evaluation,\n", + " 'policy': {f\"{r},{c}\": a for (r, c), a in sorted(policy.items())},\n", + "}\n", + "with open('results.json', 'w', encoding='utf-8') as f:\n", + " json.dump(record, f, indent=2)\n", + "\n", + "print(json.dumps({k: record[k] for k in ('training_first_100', 'training_last_100', 'evaluation')}, indent=2))\n", + "print(\"Saved results.json\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "07399630", + "metadata": {}, + "source": [ + "## Tests\n", + "\n", + "Run these cells from the project root (the folder containing `scripts/`).\n", + "The first checks the Q-learning code in this notebook. The second checks the\n", + "agent tooling: the guard hook, the checker hook, and the skill and subagent files.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6219b28d", + "metadata": {}, + "outputs": [], + "source": [ + "# Q-learning tests: environment rules, epsilon-greedy, Q-updates, evaluation,\n", + "# and an end-to-end training check. Runs from the project root.\n", + "import subprocess, sys\n", + "\n", + "result = subprocess.run(\n", + " [sys.executable, '-m', 'unittest', '-v', 'scripts.test_q_learning'],\n", + " capture_output=True, text=True)\n", + "print(result.stdout)\n", + "print(result.stderr)\n", + "print(\"Q-learning tests:\", \"PASSED\" if result.returncode == 0 else \"FAILED\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb9f3c82", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent/feature tests: the PreToolUse guard hook and the checker hook, skill, and\n", + "# subagent configuration (scripts/check_project.py and .claude/ files).\n", + "import subprocess, sys\n", + "\n", + "result = subprocess.run(\n", + " [sys.executable, '-m', 'unittest', '-v',\n", + " 'scripts.test_guard_tools', 'scripts.test_code_features'],\n", + " capture_output=True, text=True)\n", + "print(result.stdout)\n", + "print(result.stderr)\n", + "print(\"Agent/feature tests:\", \"PASSED\" if result.returncode == 0 else \"FAILED\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/q-learning-9b/q_learning_demo.py b/q-learning-9b/q_learning_demo.py new file mode 100644 index 0000000..d8ba7a5 --- /dev/null +++ b/q-learning-9b/q_learning_demo.py @@ -0,0 +1,72 @@ +"""Prepare a Lab 5 checkout for the Project 2 Q-learning demo.""" + +import argparse +import ast +import json +from pathlib import Path +import subprocess +import sys + + +NOTEBOOK = Path("q_learning.ipynb") +LEARNER = Path("learning_todos.py") +LEARNER_CELL = "learning-todos" +TRAINING_CELLS = ("imports", "environment", "parameters", LEARNER_CELL, + "policy-display", "cacc3ebd", "45565a49") + + +def write_notebook(notebook): + NOTEBOOK.write_text(json.dumps(notebook, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + + +def prepare(notebook, cell): + lines = "".join(cell["source"]).splitlines(keepends=True) + functions = ast.parse("".join(lines)).body + for func in reversed(functions): + if not isinstance(func, ast.FunctionDef) or not ast.get_docstring(func): + raise ValueError("Expected documented learner functions in the notebook.") + docstring = func.body[0] + lines[docstring.end_lineno:func.end_lineno] = [ + f' raise NotImplementedError("TODO: implement {func.name}")\n', + ] + with LEARNER.open("x", encoding="utf-8") as output: + output.writelines(lines) + cell["source"] = lines + for entry in notebook["cells"]: + if entry["cell_type"] == "code": + entry["execution_count"] = None + entry["outputs"] = [] + write_notebook(notebook) + Path("results.json").unlink(missing_ok=True) + + +def sync(notebook, cell): + source = LEARNER.read_text(encoding="utf-8") + ast.parse(source, filename=str(LEARNER)) + cell["source"] = source.splitlines(keepends=True) + write_notebook(notebook) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("prepare", "sync", "verify")) + args = parser.parse_args() + notebook = json.loads(NOTEBOOK.read_text(encoding="utf-8")) + cells = {cell.get("id"): cell for cell in notebook["cells"]} + learner = cells[LEARNER_CELL] + if args.action == "prepare": + prepare(notebook, learner) + return + + sync(notebook, learner) + if args.action == "verify": + subprocess.run([sys.executable, "scripts/check_project.py"], check=True) + subprocess.run([sys.executable, "-m", "unittest", "scripts.test_q_learning"], check=True) + namespace = {"__name__": "q_learning_notebook"} + for cell_id in TRAINING_CELLS: + source = "".join(cells[cell_id]["source"]) + exec(compile(source, f"<{cell_id}>", "exec"), namespace) + + +if __name__ == "__main__": + main() diff --git a/q-learning-9b/requirements.txt b/q-learning-9b/requirements.txt new file mode 100644 index 0000000..12e8d2d --- /dev/null +++ b/q-learning-9b/requirements.txt @@ -0,0 +1,4 @@ +# Dependencies for q_learning.ipynb and aima.agents +numpy +ipython +ipykernel diff --git a/q-learning-9b/results.json b/q-learning-9b/results.json new file mode 100644 index 0000000..bc668c6 --- /dev/null +++ b/q-learning-9b/results.json @@ -0,0 +1,55 @@ +{ + "recorded_at": "2026-11-19T13:20:01", + "hyperparameters": { + "LIVING_REWARD": -0.04, + "GAMMA": 0.95, + "ALPHA": 0.1, + "EPSILON": 0.2, + "EVALUATION_EPSILON": 0.0, + "TRAIN_EPISODES": 2000, + "EVAL_EPISODES": 500, + "MAX_STEPS": 100, + "TRAIN_SEED": 0, + "EVAL_SEED": 1 + }, + "training_first_100": { + "episodes": 100, + "goal": 88, + "trap": 12, + "timeout": 0, + "average_reward": 8.2228, + "average_steps": 12.43 + }, + "training_last_100": { + "episodes": 100, + "goal": 100, + "trap": 0, + "timeout": 0, + "average_reward": 9.64, + "average_steps": 10.0 + }, + "evaluation": { + "episodes": 500, + "goal": 500, + "trap": 0, + "timeout": 0, + "average_reward": 9.717520000000222, + "average_steps": 8.062, + "success_rate": 1.0 + }, + "policy": { + "0,0": "Right", + "0,1": "Right", + "0,2": "Right", + "0,3": "Down", + "1,0": "Up", + "1,2": "Right", + "1,3": "Down", + "2,0": "Up", + "2,1": "Up", + "2,2": "Right", + "2,3": "Down", + "3,0": "Left", + "3,2": "Up" + } +} \ No newline at end of file diff --git a/q-learning-9b/scripts/check_project.py b/q-learning-9b/scripts/check_project.py new file mode 100644 index 0000000..2f8f491 --- /dev/null +++ b/q-learning-9b/scripts/check_project.py @@ -0,0 +1,148 @@ +"""Check Lab 5 source without executing notebook cells or training the robot.""" + +import argparse +import ast +import json +import math +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +PARAMETERS = ('LIVING_REWARD', 'GAMMA', 'ALPHA', 'EPSILON', 'EVALUATION_EPSILON') + + +def check_project(root=ROOT): + errors, notes, parameters, code = [], [], {}, [] + try: + notebook = json.loads((root / 'q_learning.ipynb').read_text(encoding='utf-8')) + if notebook.get('nbformat') != 4 or not isinstance(notebook.get('cells'), list): + raise ValueError('Expected a version 4 notebook with a cells list') + for number, cell in enumerate(notebook['cells'], 1): + if cell.get('cell_type') != 'code': + continue + source = cell.get('source', []) + if not isinstance(source, (str, list)) or (isinstance(source, list) and not all(isinstance(s, str) for s in source)): + raise ValueError(f'Cell {number} has invalid source') + text = source if isinstance(source, str) else ''.join(source) + try: + tree = ast.parse(text, filename=f'q_learning.ipynb cell {number}') + compile(tree, f'q_learning.ipynb cell {number}', 'exec') + except SyntaxError as exc: + errors.append(f'Cell {number}, line {exc.lineno}: {exc.msg}') + continue + code.append(text) + # Read literal top-level assignments; never execute notebook code. + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in PARAMETERS: + try: + parameters[target.id] = ast.literal_eval(node.value) + except (ValueError, TypeError): + errors.append(f'{target.id} must be a numeric literal for this checker') + for node in ast.walk(tree): + if isinstance(node, ast.Raise) and node.exc is not None: + exc = node.exc.func if isinstance(node.exc, ast.Call) else node.exc + if isinstance(exc, ast.Name) and exc.id == 'NotImplementedError': + notes.append(f'Unfinished TODO: cell {number}, line {node.lineno}') + if not code: + errors.append('Notebook contains no valid Python code cells') + except (OSError, ValueError, TypeError, AttributeError) as exc: + errors.append(f'Cannot read notebook: {exc}') + + for name in PARAMETERS: + value = parameters.get(name) + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + errors.append(f'{name} must be a finite number') + elif name == 'GAMMA' and not 0 <= value < 1: + errors.append('GAMMA must be at least 0 and below 1') + elif name == 'ALPHA' and not 0 < value <= 1: + errors.append('ALPHA must be above 0 and at most 1') + elif name in ('EPSILON', 'EVALUATION_EPSILON') and not 0 <= value <= 1: + errors.append(f'{name} must be between 0 and 1') + + for path in (root / 'scripts').rglob('*.py'): + try: + compile(path.read_text(encoding='utf-8'), str(path), 'exec') + except (OSError, SyntaxError) as exc: + errors.append(f'{path.relative_to(root)}: {exc}') + + required = [ + 'CLAUDE.md', 'context/environment.md', 'context/requirements.md', + 'aima/__init__.py', 'aima/agents.py', 'aima/utils.py', + '.claude/settings.json', '.claude/skills/check-vacuum/SKILL.md', + '.claude/agents/environment-specialist.md', + '.claude/agents/requirements-specialist.md', + '.claude/agents/configuration-reviewer.md', + ] + for name in required: + if not (root / name).is_file(): + errors.append(f'Missing file: {name}') + try: + settings = json.loads((root / '.claude/settings.json').read_text(encoding='utf-8')) + if not settings.get('hooks', {}).get('PostToolUse'): + errors.append('PostToolUse hook is missing from settings') + except (OSError, ValueError, AttributeError) as exc: + errors.append(f'Cannot read hook settings: {exc}') + if notes: + notes.append('The Q-learning solution is unfinished; passing checks is not a training result.') + return {'errors': errors, 'notes': notes, 'parameters': parameters} + + +def summary(report): + lines = ['Lab 5 checks: ' + ('FAIL' if report['errors'] else 'PASS')] + lines += ['ERROR: ' + item for item in report['errors']] + lines += ['NOTE: ' + item for item in report['notes']] + lines += [f'{name}={value}' for name, value in report['parameters'].items()] + lines.append('Checked syntax, parameter ranges, and required files; did not execute or train the notebook.') + return '\n'.join(lines) + + +def run_hook(): + try: + payload = json.load(sys.stdin) + if not isinstance(payload, dict): + raise ValueError('Expected an object') + data = payload.get('tool_input', {}) + if not isinstance(data, dict): + raise ValueError('Expected tool_input to be an object') + filename = data.get('file_path') or data.get('notebook_path') + if not filename: + return 0 + path = Path(filename) + if not path.is_absolute(): + path = Path(payload.get('cwd', str(ROOT))) / path + try: + relative = path.resolve().relative_to(ROOT.resolve()) + except ValueError: + return 0 + if not relative.parts or relative.parts[0] in ('aima', '.git', '.venv', 'venv'): + return 0 + if path.suffix.lower() not in ('.py', '.ipynb', '.json', '.md'): + return 0 + except (ValueError, TypeError, OSError) as exc: + print(f'Invalid Lab 5 hook input: {exc}', file=sys.stderr) + return 2 + report = check_project() + output = {'hookSpecificOutput': {'hookEventName': 'PostToolUse', 'additionalContext': summary(report)}} + if report['errors']: + # PostToolUse runs after the edit. This asks Claude to fix the errors; + # it does not undo or prevent the edit that already happened. + output.update(decision='block', reason='Lab 5 checks failed. Fix the reported errors before continuing.') + print(json.dumps(output)) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--hook', action='store_true', help='Read a Claude Code PostToolUse event from stdin') + options = parser.parse_args() + if options.hook: + return run_hook() + report = check_project() + print(summary(report)) + return 1 if report['errors'] else 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/q-learning-9b/scripts/guard_tools.py b/q-learning-9b/scripts/guard_tools.py new file mode 100644 index 0000000..9d68940 --- /dev/null +++ b/q-learning-9b/scripts/guard_tools.py @@ -0,0 +1,82 @@ +"""PreToolUse guard hook for Lab 5. + +Reads the hook JSON from stdin and: + 1. denies any access to the .env secrets file, + 2. denies edits to the separate Lab 4 submission, + 3. asks for confirmation before risky Bash commands. + +Deny = exit code 2 with the reason on stderr. Ask = JSON on stdout. +""" + +import json +import re +import sys + +SAFE_ENV_SUFFIXES = ('.example', '.sample', '.template') +ENV_NAME = re.compile(r'(?:^|[\s/\\"\'=:])\.env(?P\.[\w-]+)?(?=$|[\s/\\"\';|&<>)])', re.I) +LAB4_NAME = re.compile(r'lab[\s_-]*4', re.I) +MUTATING = re.compile( + r'(\brm\b|\bmv\b|\bcp\b|\bsed\s+-i|\btee\b|>|\bgit\s+(checkout|restore|rm|mv)\b|' + r'Remove-Item|Move-Item|Copy-Item|Set-Content|Out-File|Add-Content|Rename-Item|\bdel\b|\brmdir\b)', + re.I) +RISKY = [ + (r'\bgit\s+push\b', 'git push publishes work to the shared remote'), + (r'\bgit\s+reset\s+--hard\b', 'git reset --hard discards uncommitted work'), + (r'\bgit\s+clean\b', 'git clean deletes untracked files'), + (r'\bgit\s+checkout\s+(--\s|\.)', 'git checkout discards working-tree changes'), + (r'\bgit\s+restore\b', 'git restore discards working-tree changes'), + (r'\bgit\s+branch\s+-D\b', 'git branch -D force-deletes a branch'), + (r'\brm\s+-[a-z]*[rf][a-z]*\b', 'rm with -r/-f deletes files irreversibly'), + (r'Remove-Item\b[^|;]*-Recurse', 'Remove-Item -Recurse deletes a tree irreversibly'), + (r'\b(rmdir|rd)\s+/s\b', 'rmdir /s deletes a tree irreversibly'), +] + + +def touches_env(text): + for match in ENV_NAME.finditer(text): + suffix = (match.group('suffix') or '').lower() + if suffix not in SAFE_ENV_SUFFIXES: + return True + return False + + +def deny(reason): + print(reason, file=sys.stderr) + sys.exit(2) + + +def ask(reason): + print(json.dumps({'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'ask', + 'permissionDecisionReason': reason}})) + sys.exit(0) + + +def main(): + try: + data = json.load(sys.stdin) + except ValueError: + sys.exit(0) # unreadable input: do not block work + + tool = data.get('tool_name', '') + tool_input = data.get('tool_input') or {} + target = ' '.join(str(tool_input.get(k, '')) for k in ('file_path', 'notebook_path', 'path')) + command = str(tool_input.get('command', '')) + + if touches_env(target) or touches_env(command): + deny('Blocked: .env holds secrets and must not be read, edited, or committed.') + + if tool in ('Write', 'Edit', 'NotebookEdit') and LAB4_NAME.search(target): + deny('Blocked: the Lab 4 submission is separate and must be preserved.') + if tool in ('Bash', 'PowerShell') and LAB4_NAME.search(command) and MUTATING.search(command): + deny('Blocked: this command appears to modify the Lab 4 submission, which must be preserved.') + + if tool in ('Bash', 'PowerShell'): + for pattern, why in RISKY: + if re.search(pattern, command, re.I): + ask(f'Confirm risky command: {why}.') + + +if __name__ == '__main__': + main() diff --git a/q-learning-9b/scripts/sweep_hyperparameters.py b/q-learning-9b/scripts/sweep_hyperparameters.py new file mode 100644 index 0000000..c6362a1 --- /dev/null +++ b/q-learning-9b/scripts/sweep_hyperparameters.py @@ -0,0 +1,93 @@ +"""Compare Q-learning hyperparameters over several seeds. + +Runs the notebook's own q_learning and evaluate_agent functions (definition +cells only; the driver cell is skipped). Training seeds and evaluation seeds +are disjoint. Nothing in the notebook or results.json is modified unless +--out is given. + + python scripts/sweep_hyperparameters.py + python scripts/sweep_hyperparameters.py --alpha 0.05 0.1 0.3 --epsilon 0.1 0.2 --seeds 10 + python scripts/sweep_hyperparameters.py --out sweep_results.json +""" + +import argparse +import contextlib +import io +import itertools +import json +from pathlib import Path +import statistics +import sys + +ROOT = Path(__file__).resolve().parents[1] +NOTEBOOK = ROOT / 'q_learning.ipynb' +DEFINITION_CELLS = ('imports', 'environment', 'parameters', 'learning-todos') + + +def load_notebook(): + if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + cells = {c.get('id'): ''.join(c['source']) for c in json.loads(NOTEBOOK.read_text(encoding='utf-8'))['cells']} + namespace = {'__name__': 'q_learning_sweep'} + for cell_id in DEFINITION_CELLS: + exec(compile(cells[cell_id], f'<{cell_id}>', 'exec'), namespace) + return namespace + + +def main(): + ns = load_notebook() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--alpha', type=float, nargs='+', default=[0.05, ns['ALPHA'], 0.3]) + parser.add_argument('--epsilon', type=float, nargs='+', default=[0.05, ns['EPSILON'], 0.4]) + parser.add_argument('--gamma', type=float, nargs='+', default=[ns['GAMMA']]) + parser.add_argument('--episodes', type=int, default=ns['TRAIN_EPISODES']) + parser.add_argument('--eval-episodes', type=int, default=ns['EVAL_EPISODES']) + parser.add_argument('--seeds', type=int, default=5, help='number of independent training runs per setting') + parser.add_argument('--out', help='optional JSON file for the full table') + args = parser.parse_args() + + train_seeds = list(range(1000, 1000 + args.seeds)) + eval_seeds = list(range(2000, 2000 + args.seeds)) + assert not set(train_seeds) & set(eval_seeds) + + rows = [] + print(f"{args.seeds} seeds per setting, {args.episodes} training episodes, " + f"{args.eval_episodes} evaluation episodes, living reward {ns['LIVING_REWARD']}\n") + print(f"{'alpha':>6} {'epsilon':>8} {'gamma':>6} | {'success':>8} {'eval reward':>12} " + f"{'eval steps':>11} {'train traps/100':>16} | success range") + for alpha, epsilon, gamma in itertools.product(args.alpha, args.epsilon, args.gamma): + success, reward, steps, late_traps = [], [], [], [] + for train_seed, eval_seed in zip(train_seeds, eval_seeds): + env = ns['MDPEnvironment'](living_reward=ns['LIVING_REWARD']) + q, training = ns['q_learning'](env, args.episodes, ns['MAX_STEPS'], alpha=alpha, + gamma=gamma, epsilon=epsilon, seed=train_seed) + with contextlib.redirect_stdout(io.StringIO()): + summary = ns['evaluate_agent'](env, q, args.eval_episodes, ns['MAX_STEPS'], + seed=eval_seed, show_paths=0) + success.append(summary['success_rate']) + reward.append(summary['average_reward']) + steps.append(summary['average_steps']) + late_traps.append(sum(r['outcome'] == 'trap' for r in training[-100:])) + row = {'alpha': alpha, 'epsilon': epsilon, 'gamma': gamma, + 'success_mean': statistics.mean(success), 'success_min': min(success), + 'success_max': max(success), 'eval_reward_mean': statistics.mean(reward), + 'eval_steps_mean': statistics.mean(steps), 'late_traps_mean': statistics.mean(late_traps)} + rows.append(row) + print(f"{alpha:>6} {epsilon:>8} {gamma:>6} | {row['success_mean']:>8.1%} " + f"{row['eval_reward_mean']:>12.2f} {row['eval_steps_mean']:>11.1f} " + f"{row['late_traps_mean']:>16.1f} | {row['success_min']:.1%}-{row['success_max']:.1%}") + + best = max(rows, key=lambda r: (r['eval_reward_mean'], r['success_mean'])) + print(f"\nHighest mean evaluation reward: alpha {best['alpha']}, epsilon {best['epsilon']}, " + f"gamma {best['gamma']} ({best['eval_reward_mean']:.2f}).") + print("Differences between neighbouring settings may be within seed-to-seed noise; " + "check the success range before drawing conclusions.") + + if args.out: + Path(args.out).write_text(json.dumps({'train_seeds': train_seeds, 'eval_seeds': eval_seeds, + 'rows': rows}, indent=2), encoding='utf-8') + print(f"Saved {args.out}") + + +if __name__ == '__main__': + main() diff --git a/q-learning-9b/scripts/test_code_features.py b/q-learning-9b/scripts/test_code_features.py new file mode 100644 index 0000000..69d6fb8 --- /dev/null +++ b/q-learning-9b/scripts/test_code_features.py @@ -0,0 +1,93 @@ +"""Integration checks for the Lab 5 checker and hook, not the learning robot.""" + +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class CodeFeatureTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix='lab5-features-') + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / 'project with spaces' + self.root.mkdir() + for name in ('scripts', '.claude', 'context'): + shutil.copytree(ROOT / name, self.root / name, ignore=shutil.ignore_patterns('__pycache__')) + for name in ('CLAUDE.md', 'q_learning.ipynb'): + shutil.copy2(ROOT / name, self.root / name) + # The checker only checks presence; these fixtures never import AIMA. + (self.root / 'aima').mkdir() + for name in ('__init__.py', 'agents.py', 'utils.py'): + (self.root / 'aima' / name).write_text('', encoding='utf-8') + self.notebook = self.root / 'q_learning.ipynb' + + def run_check(self, payload=None): + args = [sys.executable, str(self.root / 'scripts/check_project.py')] + if payload is not None: + args.append('--hook') + return subprocess.run(args, input=json.dumps(payload) if payload is not None else None, + text=True, capture_output=True, timeout=20, cwd=self.root.parent) + + def append_cell(self, text): + notebook = json.loads(self.notebook.read_text(encoding='utf-8')) + notebook['cells'].append({'cell_type': 'code', 'source': [text]}) + self.notebook.write_text(json.dumps(notebook), encoding='utf-8') + + def event(self, key='notebook_path', path=None): + return {'hook_event_name': 'PostToolUse', 'tool_name': 'NotebookEdit', + 'cwd': str(self.root), 'tool_input': {key: str(path or self.notebook)}} + + def test_starter_reports_incomplete_without_running_todos(self): + self.append_cell('def unfinished_fixture():\n raise NotImplementedError("TODO")\n') + result = self.run_check() + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn('Unfinished TODO', result.stdout) + self.assertIn('did not execute or train', result.stdout) + + def test_syntax_error_fails_cli_and_requests_hook_correction(self): + self.append_cell('def broken(:\n') + self.assertEqual(self.run_check().returncode, 1) + result = self.run_check(self.event()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout)['decision'], 'block') + + def test_out_of_range_hyperparameter_fails(self): + self.append_cell('EPSILON = 1.5\n') + result = self.run_check() + self.assertEqual(result.returncode, 1) + self.assertIn('EPSILON must be between 0 and 1', result.stdout) + + def test_nonfinite_learning_rate_fails(self): + self.append_cell('ALPHA = 1e999\n') + result = self.run_check() + self.assertEqual(result.returncode, 1) + self.assertIn('ALPHA must be a finite number', result.stdout) + + def test_valid_hook_supports_both_tool_path_fields_and_spaces(self): + for key in ('file_path', 'notebook_path'): + result = self.run_check(self.event(key)) + self.assertEqual(result.returncode, 0, result.stderr) + output = json.loads(result.stdout) + self.assertNotIn('decision', output) + self.assertIn('Lab 5 checks: PASS', output['hookSpecificOutput']['additionalContext']) + + def test_hook_skips_files_outside_project(self): + result = self.run_check(self.event(path=self.root.parent / 'outside.py')) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + + def test_missing_skill_fails(self): + (self.root / '.claude/skills/check-vacuum/SKILL.md').unlink() + result = self.run_check() + self.assertEqual(result.returncode, 1) + self.assertIn('Missing file: .claude/skills/check-vacuum/SKILL.md', result.stdout) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/q-learning-9b/scripts/test_guard_tools.py b/q-learning-9b/scripts/test_guard_tools.py new file mode 100644 index 0000000..686e4ed --- /dev/null +++ b/q-learning-9b/scripts/test_guard_tools.py @@ -0,0 +1,68 @@ +"""Tests for the PreToolUse guard hook (scripts/guard_tools.py).""" + +import json +from pathlib import Path +import subprocess +import sys +import unittest + +GUARD = Path(__file__).resolve().parent / 'guard_tools.py' + + +def run_guard(tool, **tool_input): + result = subprocess.run([sys.executable, str(GUARD)], text=True, capture_output=True, timeout=10, + input=json.dumps({'tool_name': tool, 'tool_input': tool_input})) + return result.returncode, result.stdout + + +class GuardTests(unittest.TestCase): + def assertDenied(self, tool, **tool_input): + self.assertEqual(run_guard(tool, **tool_input)[0], 2, (tool, tool_input)) + + def assertAllowed(self, tool, **tool_input): + code, out = run_guard(tool, **tool_input) + self.assertEqual((code, out), (0, ''), (tool, tool_input)) + + def assertAsks(self, tool, **tool_input): + code, out = run_guard(tool, **tool_input) + self.assertEqual(code, 0) + self.assertEqual(json.loads(out)['hookSpecificOutput']['permissionDecision'], 'ask') + + def test_env_file_denied(self): + self.assertDenied('Read', file_path='C:\\proj\\.env') + self.assertDenied('Edit', file_path='proj/.env.local') + self.assertDenied('Bash', command='cat .env') + self.assertDenied('PowerShell', command='Get-Content ./.env') + self.assertDenied('Bash', command='git add .env') + + def test_env_lookalikes_allowed(self): + self.assertAllowed('Read', file_path='proj/.env.example') + self.assertAllowed('Read', file_path='proj/.envrc.md') + self.assertAllowed('Bash', command='echo environment') + self.assertAllowed('Read', file_path='proj/dotenv.py') + + def test_lab4_protected(self): + self.assertDenied('Edit', file_path='proj/Lab4/solution.py') + self.assertDenied('Write', file_path='proj/lab 4/notes.md') + self.assertDenied('Bash', command='rm -r lab4') + self.assertAllowed('Read', file_path='proj/Lab4/solution.py') + self.assertAllowed('Bash', command='ls lab4') + + def test_risky_bash_asks(self): + self.assertAsks('Bash', command='git push origin main') + self.assertAsks('Bash', command='git reset --hard HEAD~1') + self.assertAsks('Bash', command='rm -rf build') + self.assertAsks('PowerShell', command='Remove-Item build -Recurse -Force') + + def test_normal_work_allowed(self): + self.assertAllowed('Bash', command='git status') + self.assertAllowed('Edit', file_path='q_learning.ipynb') + self.assertAllowed('Read', file_path='context/environment.md') + + def test_bad_input_does_not_block(self): + result = subprocess.run([sys.executable, str(GUARD)], text=True, capture_output=True, input='not json') + self.assertEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/q-learning-9b/scripts/test_q_learning.py b/q-learning-9b/scripts/test_q_learning.py new file mode 100644 index 0000000..3523f76 --- /dev/null +++ b/q-learning-9b/scripts/test_q_learning.py @@ -0,0 +1,317 @@ +"""Tests for the Q-learning code in q_learning.ipynb. + +The notebook's definition cells are executed into a namespace; the driver cell +that trains and prints results is skipped. Run from the project root: + + python -m unittest scripts.test_q_learning -v +""" + +import contextlib +import copy +import io +import json +from pathlib import Path +import random +import re +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[1] +NOTEBOOK = ROOT / 'q_learning.ipynb' +DEFINITION_CELLS = ('imports', 'environment', 'parameters', 'learning-todos', 'policy-display') +LEARNER_CELL = 'learning-todos' + + +def cell_source(cell_id): + cells = json.loads(NOTEBOOK.read_text(encoding='utf-8'))['cells'] + for cell in cells: + if cell.get('id') == cell_id: + return ''.join(cell['source']) + raise KeyError(cell_id) + + +def load_notebook_namespace(): + if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + namespace = {'__name__': 'q_learning_notebook'} + for cell_id in DEFINITION_CELLS: + exec(compile(cell_source(cell_id), f'<{cell_id}>', 'exec'), namespace) + return namespace + + +NS = load_notebook_namespace() +MDPEnvironment, Agent = NS['MDPEnvironment'], NS['Agent'] +ACTIONS = NS['ACTIONS'] + + +def new_robot(env): + robot = Agent(lambda percept: None) + env.reset(robot) + return robot + + +class ScriptedEnv: + """Deterministic stand-in environment: transitions[(state, action)] = (next, reward, done).""" + + def __init__(self, transitions, start='A'): + self.transitions, self.start, self.state, self.done = transitions, start, start, False + + def reset(self, robot): + self.state, self.done = self.start, False + return self.state + + def execute_action(self, robot, action): + self.state, reward, self.done = self.transitions[(self.state, action)] + return self.state, reward, self.done + + +def every_action(state, nxt, reward, done): + return {(state, a): (nxt, reward, done) for a in ACTIONS} + + +class EnvironmentTests(unittest.TestCase): + def test_reset_returns_start_and_clears_score(self): + env = MDPEnvironment() + robot = Agent(lambda p: None) + robot.performance = 5 + self.assertEqual(env.reset(robot), (0, 0)) + self.assertEqual((robot.location, robot.performance, robot.alive, env.done), ((0, 0), 0, True, False)) + + def test_ordinary_and_blocked_moves_get_living_reward(self): + env = MDPEnvironment(living_reward=-0.04) + robot = new_robot(env) + for _ in range(30): # at the top-left corner, Up/Left are blocked or slip into walls + loc, reward, done = env.execute_action(robot, 'Up') + self.assertEqual((reward, done), (-0.04, False)) + self.assertIn(loc, [(0, 0), (0, 1)]) + robot.location = (0, 0) + + def test_terminal_cells_pay_exactly_their_reward_and_end_episode(self): + for cell, expected_alive in ((10, True), (-1, False)): + env = MDPEnvironment(env=[['start', cell]], living_reward=-0.5) + env._attempt = lambda agent, intended, *_: intended(agent) # no slips + robot = new_robot(env) + loc, reward, done = env.execute_action(robot, 'Right') + self.assertEqual((loc, reward, done, robot.alive), ((0, 1), cell, True, expected_alive)) + + def test_action_after_episode_end_raises(self): + env = MDPEnvironment(env=[['start', 10]]) + env._attempt = lambda agent, intended, *_: intended(agent) + robot = new_robot(env) + env.execute_action(robot, 'Right') + with self.assertRaises(RuntimeError): + env.execute_action(robot, 'Right') + env.reset(robot) # reset makes it usable again + env.execute_action(robot, 'Right') + + def test_unknown_action_raises(self): + env = MDPEnvironment() + with self.assertRaises(ValueError): + env.execute_action(new_robot(env), 'Jump') + + def test_obstacle_is_never_entered(self): + env = MDPEnvironment() + random.seed(3) + robot = new_robot(env) + for _ in range(500): + if env.done: + env.reset(robot) + loc, _, _ = env.execute_action(robot, random.choice(ACTIONS)) + self.assertNotEqual(loc, (1, 1)) + + def test_slip_probabilities_are_80_10_10(self): + env = MDPEnvironment() + random.seed(0) + counts = {} + n = 20000 + for _ in range(n): + robot = new_robot(env) + robot.location = (2, 2) + loc, _, _ = env.execute_action(robot, 'Right') + env.done = False + counts[loc] = counts.get(loc, 0) + 1 + self.assertEqual(set(counts), {(2, 3), (1, 2), (3, 2)}) + self.assertAlmostEqual(counts[(2, 3)] / n, 0.8, delta=0.015) + self.assertAlmostEqual(counts[(1, 2)] / n, 0.1, delta=0.015) + self.assertAlmostEqual(counts[(3, 2)] / n, 0.1, delta=0.015) + + +class ChooseActionTests(unittest.TestCase): + choose = staticmethod(NS['choose_action']) + + def test_greedy_picks_best_action(self): + q = {'s': {'Up': 0.0, 'Down': 1.0, 'Left': 0.5, 'Right': -1.0}} + rng = random.Random(0) + self.assertTrue(all(self.choose(q, 's', 0.0, rng) == 'Down' for _ in range(50))) + + def test_ties_are_broken_randomly_among_best_only(self): + q = {'s': {'Up': 1.0, 'Down': 1.0, 'Left': 0.0, 'Right': 1.0}} + rng = random.Random(0) + picks = {self.choose(q, 's', 0.0, rng) for _ in range(200)} + self.assertEqual(picks, {'Up', 'Down', 'Right'}) + + def test_epsilon_one_explores_all_actions(self): + q = {'s': {'Up': 9.0, 'Down': 0.0, 'Left': 0.0, 'Right': 0.0}} + rng = random.Random(0) + self.assertEqual({self.choose(q, 's', 1.0, rng) for _ in range(200)}, set(ACTIONS)) + + def test_exploration_rate_roughly_matches_epsilon(self): + q = {'s': {'Up': 9.0, 'Down': 0.0, 'Left': 0.0, 'Right': 0.0}} + rng = random.Random(1) + n = 20000 + non_greedy = sum(self.choose(q, 's', 0.2, rng) != 'Up' for _ in range(n)) / n + self.assertAlmostEqual(non_greedy, 0.2 * 0.75, delta=0.02) # 3 of 4 random picks are non-greedy + + def test_unseen_state_falls_back_to_random(self): + rng = random.Random(0) + self.assertEqual({self.choose({}, 'new', 0.0, rng) for _ in range(200)}, set(ACTIONS)) + + +class OutcomeTests(unittest.TestCase): + def test_labels(self): + outcome_of = NS['outcome_of'] + self.assertEqual(outcome_of(10, True), 'goal') + self.assertEqual(outcome_of(-1, True), 'trap') + self.assertEqual(outcome_of(-0.04, False), 'timeout') + + +class QUpdateTests(unittest.TestCase): + q_learning = staticmethod(NS['q_learning']) + + def test_terminal_update_does_not_bootstrap(self): + env = ScriptedEnv(every_action('A', 'END', 10, True)) + q, results = self.q_learning(env, 1, 10, alpha=0.5, gamma=0.9, epsilon=0.0, seed=0) + updated = [a for a in ACTIONS if q['A'][a] != 0.0] + self.assertEqual(len(updated), 1) + self.assertAlmostEqual(q['A'][updated[0]], 0.5 * 10) # alpha * (reward - 0) + self.assertNotIn('END', q) # terminal state never gets its own entry + self.assertEqual(results[0]['outcome'], 'goal') + + def test_nonterminal_update_bootstraps_from_next_state_max(self): + transitions = {**every_action('A', 'B', -0.04, False), **every_action('B', 'END', 10, True)} + env = ScriptedEnv(transitions) + q, _ = self.q_learning(env, 2, 10, alpha=1.0, gamma=0.9, epsilon=0.0, seed=0) + # episode 1: Q(B,*) becomes 10 for the chosen action; episode 2: Q(A,a) = -0.04 + 0.9 * 10 + self.assertAlmostEqual(max(q['B'].values()), 10.0) + self.assertAlmostEqual(max(q['A'].values()), -0.04 + 0.9 * 10.0) + + def test_step_limit_gives_timeout(self): + env = ScriptedEnv(every_action('A', 'A', -0.04, False)) + q, results = self.q_learning(env, 3, 7, seed=0) + self.assertEqual([r['outcome'] for r in results], ['timeout'] * 3) + self.assertEqual([r['steps'] for r in results], [7] * 3) + self.assertAlmostEqual(results[0]['reward'], -0.04 * 7) + + def test_trap_outcome_recorded(self): + env = ScriptedEnv(every_action('A', 'T', -1, True)) + _, results = self.q_learning(env, 2, 10, seed=0) + self.assertEqual([r['outcome'] for r in results], ['trap', 'trap']) + self.assertEqual(results[0]['reward'], -1) + + def test_same_seed_is_reproducible_and_different_seed_differs(self): + env = MDPEnvironment() + q1, r1 = self.q_learning(env, 200, 100, seed=5) + q2, r2 = self.q_learning(env, 200, 100, seed=5) + q3, r3 = self.q_learning(env, 200, 100, seed=6) + self.assertEqual((q1, r1), (q2, r2)) + self.assertNotEqual(q1, q3) + + def test_one_result_per_episode(self): + _, results = self.q_learning(MDPEnvironment(), 50, 100, seed=0) + self.assertEqual([r['episode'] for r in results], list(range(1, 51))) + + +class EvaluationTests(unittest.TestCase): + def trained(self): + return NS['q_learning'](MDPEnvironment(), 1500, 100, seed=NS['TRAIN_SEED'])[0] + + def test_evaluation_does_not_change_q_values(self): + q = self.trained() + before = copy.deepcopy(q) + with contextlib.redirect_stdout(io.StringIO()): + NS['evaluate_agent'](MDPEnvironment(), q, 100, 100, seed=NS['EVAL_SEED']) + self.assertEqual(q, before) + + def test_evaluation_summary_is_consistent(self): + q = self.trained() + with contextlib.redirect_stdout(io.StringIO()): + s = NS['evaluate_agent'](MDPEnvironment(), q, 100, 100, seed=NS['EVAL_SEED']) + self.assertEqual(s['goal'] + s['trap'] + s['timeout'], s['episodes']) + self.assertAlmostEqual(s['success_rate'], s['goal'] / 100) + + def test_evaluation_is_greedy(self): + # With a hand-built Q-table that always says Right and a no-slip world, the path is fixed. + env = MDPEnvironment(env=[['start', 0, 10]]) + env._attempt = lambda agent, intended, *_: intended(agent) + q = {(0, 0): {'Up': 0, 'Down': 0, 'Left': 0, 'Right': 1.0}, + (0, 1): {'Up': 0, 'Down': 0, 'Left': 0, 'Right': 1.0}} + out = io.StringIO() + with contextlib.redirect_stdout(out): + s = NS['evaluate_agent'](env, q, 20, 10, seed=0, show_paths=1) + self.assertEqual((s['goal'], s['average_steps']), (20, 2.0)) + self.assertIn('Right -> (0, 1) | Right -> (0, 2)', out.getvalue()) + + def test_train_and_eval_seeds_are_separate(self): + self.assertNotEqual(NS['TRAIN_SEED'], NS['EVAL_SEED']) + + +class PolicyTests(unittest.TestCase): + def test_derive_policy_takes_argmax_per_state(self): + q = {(0, 0): {'Up': 0.0, 'Down': 0.1, 'Left': 0.0, 'Right': 0.9}} + self.assertEqual(NS['derive_policy'](q), {(0, 0): 'Right'}) + + def test_print_policy_marks_walls_terminals_and_unvisited(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + NS['print_policy']({(0, 0): 'Right'}) + text = out.getvalue() + for token in ('R', 'X', 'TRAP', 'RECHARGE', '?'): + self.assertIn(token, text) + + +class LearningIntegrationTests(unittest.TestCase): + """End-to-end check with the notebook's own hyperparameters.""" + + @classmethod + def setUpClass(cls): + env = MDPEnvironment(living_reward=NS['LIVING_REWARD']) + cls.q, cls.training = NS['q_learning']( + env, NS['TRAIN_EPISODES'], NS['MAX_STEPS'], alpha=NS['ALPHA'], gamma=NS['GAMMA'], + epsilon=NS['EPSILON'], seed=NS['TRAIN_SEED']) + with contextlib.redirect_stdout(io.StringIO()): + cls.summary = NS['evaluate_agent'](env, cls.q, NS['EVAL_EPISODES'], NS['MAX_STEPS'], + seed=NS['EVAL_SEED']) + + def test_learned_policy_reaches_goal_most_of_the_time(self): + self.assertGreaterEqual(self.summary['success_rate'], 0.9) + + def test_learned_policy_avoids_the_trap_cell(self): + self.assertLessEqual(self.summary['trap'], 0.05 * self.summary['episodes']) + + def test_reward_improves_from_early_to_late_training(self): + early = sum(r['reward'] for r in self.training[:100]) / 100 + late = sum(r['reward'] for r in self.training[-100:]) / 100 + self.assertGreater(late, early) + + def test_policy_next_to_goal_heads_to_goal(self): + policy = NS['derive_policy'](self.q) + self.assertEqual(policy[(2, 3)], 'Down') + + +class SpecComplianceTests(unittest.TestCase): + def test_hyperparameters_in_valid_ranges(self): + self.assertTrue(0 < NS['ALPHA'] <= 1) + self.assertTrue(0 <= NS['GAMMA'] <= 1) + self.assertTrue(0 <= NS['EPSILON'] <= 1) + self.assertEqual(NS['EVALUATION_EPSILON'], 0.0) + self.assertLess(NS['LIVING_REWARD'], 0) + self.assertGreater(NS['MAX_STEPS'], 0) + + def test_learner_code_never_reads_simulator_internals(self): + source = cell_source(LEARNER_CELL) + for forbidden in ('.status', '_attempt', 'find_recharge_location', 'default_world', 'living_reward'): + self.assertIsNone(re.search(re.escape(forbidden), source), forbidden) + + +if __name__ == '__main__': + unittest.main() diff --git a/q-learning-9b/scripts/verify_results.py b/q-learning-9b/scripts/verify_results.py new file mode 100644 index 0000000..fad410d --- /dev/null +++ b/q-learning-9b/scripts/verify_results.py @@ -0,0 +1,119 @@ +"""Sanity-check results.json against the environment rules. + +Verification only: this script knows the grid and slip rules and solves the +environment with value iteration to compare against the learned policy. The +learner in q_learning.ipynb never sees any of this. + + python scripts/verify_results.py [path/to/results.json] + +Exit code 1 if the recorded numbers are internally inconsistent, else 0. +Policy differences from optimal are reported but are not failures. +""" + +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +GRID = [['s', 0, 0, 0], [0, 'X', 0, 0], [0, 0, 0, 0], [0, -1, 0, 10]] +MOVES = {'Up': (-1, 0), 'Down': (1, 0), 'Left': (0, -1), 'Right': (0, 1)} +SIDEWAYS = {'Up': ('Right', 'Left'), 'Down': ('Left', 'Right'), + 'Left': ('Up', 'Down'), 'Right': ('Up', 'Down')} +TERMINAL = {(3, 3): 10, (3, 1): -1} +TOL = 1e-6 + + +def step(state, action): + r, c = state[0] + MOVES[action][0], state[1] + MOVES[action][1] + ok = 0 <= r < 4 and 0 <= c < 4 and GRID[r][c] != 'X' + return (r, c) if ok else state + + +def action_value(state, action, values, gamma, living): + total = 0.0 + for prob, actual in ((0.8, action), (0.1, SIDEWAYS[action][0]), (0.1, SIDEWAYS[action][1])): + nxt = step(state, actual) + total += prob * (TERMINAL[nxt] if nxt in TERMINAL else living + gamma * values[nxt]) + return total + + +def solve(gamma, living): + cells = [(r, c) for r in range(4) for c in range(4) + if GRID[r][c] != 'X' and (r, c) not in TERMINAL] + values = {s: 0.0 for s in cells} + for _ in range(3000): + values = {s: max(action_value(s, a, values, gamma, living) for a in MOVES) for s in cells} + best = {s: max(MOVES, key=lambda a: action_value(s, a, values, gamma, living)) for s in cells} + return values, best + + +def policy_value(policy, gamma, living): + values = {s: 0.0 for s in policy} + for _ in range(3000): + values = {s: action_value(s, policy[s], values, gamma, living) for s in policy} + return values + + +def main(): + path = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / 'results.json' + data = json.loads(path.read_text(encoding='utf-8')) + hp = data['hyperparameters'] + gamma, living = hp['GAMMA'], hp['LIVING_REWARD'] + problems = [] + + print(f"Results recorded at {data.get('recorded_at', '?')}") + for name in ('training_first_100', 'training_last_100', 'evaluation'): + row = data[name] + n = row['episodes'] + if row['goal'] + row['trap'] + row['timeout'] != n: + problems.append(f'{name}: outcome counts do not add up to {n}') + if row['timeout'] == 0 and row['trap'] == 0: # all goals: reward is fixed by steps + expected = 10 + living * (row['average_steps'] - 1) + if abs(row['average_reward'] - expected) > 1e-3: + problems.append(f"{name}: reward {row['average_reward']:.4f} != {expected:.4f} implied by steps") + print(f" {name}: goal {row['goal']}, trap {row['trap']}, timeout {row['timeout']}, " + f"avg reward {row['average_reward']:.3f}, avg steps {row['average_steps']:.2f}") + + ev = data['evaluation'] + if abs(ev['success_rate'] - ev['goal'] / ev['episodes']) > TOL: + problems.append('evaluation: success_rate does not match goal count') + if ev['average_steps'] < 6: + problems.append('evaluation: average steps below the 6-step shortest path') + if ev['average_reward'] > 10 + living * 5 + TOL: + problems.append('evaluation: average reward above the best possible 6-step reward') + + optimal_values, optimal = solve(gamma, living) + learned = {tuple(int(x) for x in k.split(',')): v for k, v in data['policy'].items()} + missing = sorted(set(optimal) - set(learned)) + if missing: + print(f' Unvisited cells with no learned action: {missing}') + learned_values = policy_value({s: a for s, a in learned.items() if s in optimal}, gamma, living) \ + if not missing else None + + print('\nLearned action vs optimal action (discounted values):') + differ = 0 + for s in sorted(optimal): + if s not in learned: + continue + same = learned[s] == optimal[s] + differ += not same + note = '' if same else (f" differs: optimal {optimal[s]} " + f"({action_value(s, optimal[s], optimal_values, gamma, living):.2f}) vs " + f"{learned[s]} ({action_value(s, learned[s], optimal_values, gamma, living):.2f})") + print(f' {s}: {learned[s]}{note}') + print(f'\n{differ} of {len(learned)} learned actions differ from optimal.') + print(f'Optimal value at start: {optimal_values[(0, 0)]:.3f}') + if learned_values: + print(f'Learned policy value at start: {learned_values[(0, 0)]:.3f}') + + if problems: + print('\nINCONSISTENT:') + for p in problems: + print(' -', p) + return 1 + print('\nResults are internally consistent.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/test_mini_coding_agent.py b/tests/test_mini_coding_agent.py index 5c62693..f16720b 100644 --- a/tests/test_mini_coding_agent.py +++ b/tests/test_mini_coding_agent.py @@ -30,6 +30,52 @@ def build_agent(tmp_path, outputs, **kwargs): ) +def test_skill_matches_and_resets_on_the_next_request(tmp_path): + folder = tmp_path / "skill" / "check-vacuum" + folder.mkdir(parents=True) + (folder / "skill.md").write_text( + "---\nname: check-vacuum\ndescription: Review vacuum robot context.\n---\n" + "Read the persistent project decisions.", + encoding="utf-8", + ) + agent = build_agent(tmp_path, ["Checked.", "Hello."]) + + agent.ask("Review vacuum robot context") + assert "Read the persistent project decisions." in agent.prompt("Continue") + + agent.ask("Hello") + assert "Active skill:" not in agent.prompt("Hello") + events = [json.loads(line) for line in agent.skill_log.path.read_text().splitlines()] + assert [event["event"] for event in events] == [ + "prompt", "similarity_check", "skill_executed", "prompt", "similarity_check", + ] + assert events[0]["prompt"] == "Review vacuum robot context" + assert events[1]["scores"]["check-vacuum"] >= events[1]["threshold"] + assert events[2]["description"] == "Review vacuum robot context." + assert events[4]["selected"] is None + + +def test_use_skill_logs_the_mid_run_trigger(tmp_path): + folder = tmp_path / ".claude" / "skills" / "check-vacuum" + folder.mkdir(parents=True) + (folder / "SKILL.md").write_text("Check the vacuum context.", encoding="utf-8") + agent = build_agent(tmp_path, [ + '{"name":"use_skill","args":{"name":"check-vacuum"}}', + "Checked.", + ]) + + agent.ask("Hello") + events = [json.loads(line) for line in agent.skill_log.path.read_text().splitlines()] + trigger, match, execution = events[-3:] + assert trigger["source"] == "tool" + assert trigger["prompt"] == "check-vacuum" + assert match["method"] == "exact_name" + assert match["selected"] == "check-vacuum" + assert execution["skill"] == "check-vacuum" + assert execution["trigger"] == "tool" + assert "Check the vacuum context." in agent.prompt("Continue") + + def test_agent_runs_tool_then_final(tmp_path): (tmp_path / "hello.txt").write_text("alpha\nbeta\n", encoding="utf-8") agent = build_agent( @@ -234,6 +280,30 @@ def test_repeated_identical_tool_call_is_rejected(tmp_path): assert result == "error: repeated identical tool call for list_files; choose a different tool or return a final answer" +@pytest.mark.parametrize("name,args", [ + ("run_shell", {"command": "python scripts/check_project.py"}), + ("patch_file", {"path": "README.md", "old_text": "demo", "new_text": "updated"}), +]) +def test_approval_is_not_repeated_until_a_new_request_or_edit(tmp_path, monkeypatch, name, args): + agent = build_agent(tmp_path, [], approval_policy="ask") + approvals = [] + monkeypatch.setattr(agent, "approve", lambda name, args: approvals.append(name) or False) + def record(role, tool="", tool_args=None, content=""): + agent.record({"role": role, "name": tool, "args": tool_args, "content": content}) + + record("user") + record("tool", name, args, agent.run_tool(name, args)) + record("tool", "check-vacuum", {}, "error: unknown tool 'check-vacuum'") + assert "repeated identical tool call" in agent.run_tool(name, args) + assert len(approvals) == 1 + + record("tool", "patch_file", {"path": "learning_todos.py"}, "patched learning_todos.py") + assert "approval denied" in agent.run_tool(name, args) + record("user") + assert "approval denied" in agent.run_tool(name, args) + assert len(approvals) == 3 + + def test_welcome_screen_keeps_box_shape_for_long_paths(tmp_path): deep = tmp_path / "very" / "long" / "path" / "for" / "the" / "mini" / "agent" / "welcome" / "screen" deep.mkdir(parents=True)