diff --git a/README.md b/README.md index f404701..2aec71b 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ cd pipelinerl Create the environments with dependencies. ```bash conda create -n pipeline-rl -y python=3.11 -conda run --no-capture-output -n pipeline-rl pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cu121 -conda run --no-capture-output -n pipeline-rl pip install -r requirements.txt --no-build-isolation +conda run --no-capture-output -n pipeline-rl pip install torch==2.6.0 +conda run --no-capture-output -n pipeline-rl pip install -e . --no-build-isolation ``` By default Pipeline-RL will use the file system as the medium for streaming the generated data to the trainer processes. This works on one node, but the files can get quite large. To use Redis instead you will need to install the Redis server in the same conda environment: diff --git a/conf/actor/web.yaml b/conf/actor/web.yaml new file mode 100644 index 0000000..f331e87 --- /dev/null +++ b/conf/actor/web.yaml @@ -0,0 +1,109 @@ +log_each_n_secs: 10 +llm_max_rollouts: 128 +rollout_workers: 1 +rollout_policy: pipelinerl.tapeagents_rollouts.generate_rollout + +environment: + _target_: tapeagents.mcp.MCPEnvironment + config_path: conf/mcp/web.json + +llm: + _target_: tapeagents.llms.LiteLLM + model_name: o4-mini-2025-04-16 + use_cache: true + context_size: 200000 + parameters: + temperature: 1 + max_completion_tokens: 16000 + +agent: + _target_: tapeagents.agent.Agent + name : web_agent + llms: + default: ${llm} + templates: + system_prompt: | + You are an expert AI Agent trained to assist users with complex information processing tasks. + Your role is to understand user queries and respond in a helpful and accurate manner. + Keep your replies concise and direct. Prioritize clarity and avoid over-elaboration. + Do not express emotions or opinions about user questions. + allowed_tools: | + You have access to the following tools: + {tools_description} + thought_format: | + Important! Respond with the plain text, do not include any JSON or code. + Do not output anything besides what I asked in this message. + allowed_steps: | + You have access to the following tools: + {tools_description} + You are allowed to produce ONLY steps with the following JSON schemas: + {allowed_steps} + Do not reproduce the schema when producing steps; use it as a reference. + format: > + Output only a single JSON dict or a single JSON list. + DO NOT OUTPUT ANYTHING BESIDES THE JSON! DO NOT PLACE ANY COMMENTS INSIDE THE JSON. + It will break the system that processes the output. + + nodes: + - _target_: tapeagents.nodes.StandardNode + name: plan + system_prompt: ${agent.templates.system_prompt} + guidance: | + Write a concise multi-step plan explaining which steps should be performed to find the answer for the given task. + Be specific about how each step should be performed. Only describe the intended actions here, do not perform them yet. + Consider that next steps may depend on results of previous steps, so include conditional branching using "if" statements where needed. + Start with the title "Plan". Every step should have short name and description. + ${agent.templates.thought_format} + steps_prompt: ${agent.templates.allowed_tools} + + - _target_: tapeagents.nodes.StandardNode + name: reflect + system_prompt: ${agent.templates.system_prompt} + guidance: | + Observe the current state of the task and produce the reflection text strictly following these rules: + 1. Evaluate the action's success, explain its impact on the task and our plan, + 2. If the last action was not successful, describe errors and the possible reasons for failure. + 3. List the next steps to accomplish the current plan step and propose single next immediate action. + 4. When proposing webpage interactions: + - Always accept cookie and close popups first before interacting + - If the last action was not successful, check if the target element is visible and use scrolling if it is not. + 5. Describe the expected effect of the proposed action. + ${agent.templates.thought_format} + steps_prompt: ${agent.templates.allowed_tools} + + - _target_: tapeagents.nodes.StandardNode + name: act + system_prompt: ${agent.templates.system_prompt} + guidance: Then produce single function call for the next step. If the answer is ready, call FinalStep function. + steps: + - tapeagents.steps.ReasoningThought + - tapeagents.core.FinalStep + use_known_actions: true + use_function_calls: true + next_node: act + + - _target_: tapeagents.nodes.StandardNode + name: summarize + system_prompt: ${agent.templates.system_prompt} + guidance: | + Summarize last observation. If its an image, thoroughly describe it with all details. + Describe the results of the last action and observed changes. Discuss its impact on the task and our plan. + Do not hallucinate or make up any information, only describe what you see in the observation. + Do not guess or assume action effects, describe only visible changes. + ${agent.templates.thought_format} + steps_prompt: ${agent.templates.allowed_tools} + next_node: reflect + +split: validation +batch: 2 +retry_unsolved: true + +only_tasks: #[] # list of (level, task_num) +- [1, 0] +- [1, 1] +- [1, 2] +- [1, 3] +- [1, 4] +- [1, 5] +- [1, 6] +- [1, 7] \ No newline at end of file diff --git a/conf/base.yaml b/conf/base.yaml index b56d3fb..d05354a 100644 --- a/conf/base.yaml +++ b/conf/base.yaml @@ -38,11 +38,16 @@ finetune: weight_update_interval: 1 pop_old_data: ${..pop_old_data} actor: + log_each_n_secs: 0 llm_max_rollouts: 128 rollout_workers: 1 -verifier: - host: localhost - port: 7777 + rollout_policy: pipelinerl.math.rollouts.generate_math_rollout + discount_factor: 1 + system_prompt: Please reason step by step, and put your final answer within \boxed{}. + task_template: |- + {task} +environment: + _target_: pipelinerl.math.verifier_api.MathEnvironment preprocess: input: actor output: training_data @@ -81,19 +86,18 @@ vllm_config: generation-config: vllm world: - actors: 1 - preprocessors: 1 + replicas: 1 actor_fraction: 4 preprocessor_fraction: 0 finetune_fraction: 4 - actor_group_port: 9000 + env_replicas: 2 -# changed -system_prompt: Please reason step by step, and put your final answer within \boxed{}. -task_template: |- - {task} + actor_group_port: 9000 + environment_start_port: 7777 +# this will be autocreated based on the config +jobs: [] eval_every_n_versions: 78000 @@ -115,7 +119,8 @@ force_restart: false pop_old_data: true max_lag: null attempts: 8 -discount_factor: 1 + +dataset_loader: pipelinerl.math.load_datasets.load_datasets train_dataset_names: - open_reasoner_zero_57k - open_reasoner_zero_extended_72k @@ -130,6 +135,10 @@ debug: streams_from: null place_inference_workers: true +me: + # Which job is this one? This will be autopopulated + job_idx: null + hydra: run: dir: ${output_dir} \ No newline at end of file diff --git a/conf/counting.yaml b/conf/counting.yaml new file mode 100644 index 0000000..79a8237 --- /dev/null +++ b/conf/counting.yaml @@ -0,0 +1,19 @@ +defaults: + - base +finetune: + seq_length: 4000 + gradient_accumulation_passes: 1024 +llm: + parameters: + max_tokens: 1000 +test_llm: + parameters: + max_tokens: 1000 +actor: + rollout_policy: pipelinerl.counting.counting.generate_counting_rollout +environment: null +dataset_loader: pipelinerl.counting.counting.load_problems +train_dataset_names: + - train_counting_problems +test_dataset_names: + - test_counting_problems diff --git a/conf/debug.yaml b/conf/debug.yaml new file mode 100644 index 0000000..2a3ab03 --- /dev/null +++ b/conf/debug.yaml @@ -0,0 +1,27 @@ +defaults: + - base + - override streams: redis + - _self_ + +finetune: + seq_length: 5000 + gradient_accumulation_passes: 1024 + +llm: + parameters: + max_tokens: 4096 + +test_llm: + parameters: + max_tokens: 4096 + +# debug: + # mode: open_loop + +output_dir: results/debug_4gpu_7b/${now:%Y_%m_%d}/${now:start_at_%H_%M_%S} + +# model_path: Qwen/Qwen2.5-0.5B + +# vllm_config: +# vllm_kwargs: +# enforce_eager: "" \ No newline at end of file diff --git a/conf/mcp/web.json b/conf/mcp/web.json new file mode 100644 index 0000000..47c63d5 --- /dev/null +++ b/conf/mcp/web.json @@ -0,0 +1,23 @@ +{ + "mcpServers": { + "serper-search": { + "command": "uv", + "args": ["run", "tapeagents/tools/mcp_servers/web_search.py"], + "env": {"SERPER_API_KEY": ""} + }, + "fetch": { + "command": "uvx", + "args": [ + "mcp-server-fetch" + ] + }, + "python_exec": { + "command": "npx", + "args": [ + "-y", + "@pydantic/mcp-run-python", + "stdio" + ] + } + } +} \ No newline at end of file diff --git a/pipelinerl/async_llm.py b/pipelinerl/async_llm.py index 4a482ba..303b191 100644 --- a/pipelinerl/async_llm.py +++ b/pipelinerl/async_llm.py @@ -1,7 +1,7 @@ import logging import aiohttp -from tapeagents.core import LLMCall, LLMOutput, Prompt, TokenLogprob +from tapeagents.core import LLMCall, LLMOutput, Prompt from tapeagents.llms.trainable import TrainableLLM diff --git a/pipelinerl/cot_math_agent.py b/pipelinerl/cot_math_agent.py deleted file mode 100644 index ccf6188..0000000 --- a/pipelinerl/cot_math_agent.py +++ /dev/null @@ -1,105 +0,0 @@ -import logging -from typing import Annotated, Any, Generator, Literal, TypeAlias, Union - -from pydantic import Field - -from tapeagents.agent import Agent -from tapeagents.core import ( - LLMOutputParsingFailureAction, - Observation, - Prompt, - Step, - Tape, - Thought, -) -from tapeagents.llms import LLM -from tapeagents.nodes import StandardNode - -logger = logging.getLogger(__name__) - - -class Task(Observation): - kind: Literal["task"] = "task" - task: str - template: str = Field( - description="Template for the task. Should contain a {task} placeholder for the task text.", default="{task}" - ) - - def llm_view(self, indent: int | None = 2) -> str: - return self.template.format(task=self.task) - - -class ReasoningThought(Thought): - """ - Thoughts produced by the agent during the reasoning process. - """ - - kind: Literal["reasoning_thought_with_value"] = "reasoning_thought_with_value" - reasoning: str = Field(description="chain of thoughts") - - -MathAgentStep: TypeAlias = Annotated[ - ReasoningThought, - Field(discriminator="kind"), -] - -RLMathTape = Tape[ - None, - Union[ - Task, - ReasoningThought, - LLMOutputParsingFailureAction, - ], -] - - -class ReasoningNode(StandardNode): - max_prompt_length: int = 1024 - - def parse_completion(self, completion: str) -> Generator[Step, None, None]: - try: - step = ReasoningThought(reasoning=completion) - except Exception as e: - logger.info(f"Failed to parse agent output: {completion}\n\nError: {e}") - yield LLMOutputParsingFailureAction( - error=f"Failed to parse agent output: {completion}\n\nError: {e}", llm_output=completion - ) - return - yield step - - def make_prompt(self, agent: Any, tape: Tape) -> Prompt: - messages = [] - if self.system_prompt: - messages.append({"role": "system", "content": self.system_prompt}) - - # the tape is only step long and it is the task - task = tape.steps[0] - assert isinstance(task, Task), f"Expected a Task, got {task.__class__.__name__}" - messages.append({"role": "user", "content": task.llm_view()}) - # messages = self.tape_to_messages(cleaned_tape, steps_description) - prompt_token_ids = agent.llm.tokenizer.apply_chat_template( - messages, add_special_tokens=True, add_generation_prompt=True - ) - prompt_token_ids = prompt_token_ids[-self.max_prompt_length :] - return Prompt(messages=messages, token_ids=prompt_token_ids) - - -#### Agent and Environment #### -class CoTMathAgent(Agent): - @classmethod - def create(cls, system_prompt: str, llm: LLM, max_prompt_length: int): - agent = super().create( - llm, - nodes=[ - ReasoningNode( - name="cot", - agent_step_cls=MathAgentStep, - system_prompt=system_prompt if system_prompt else "", - max_prompt_length=max_prompt_length, - ), - ], - max_iterations=1, - ) - agent.store_llm_calls = True - agent.llm.load_tokenizer() - return agent diff --git a/pipelinerl/counting/counting.py b/pipelinerl/counting/counting.py new file mode 100644 index 0000000..85d6309 --- /dev/null +++ b/pipelinerl/counting/counting.py @@ -0,0 +1,91 @@ +import os +import json +import re +import time +import aiohttp +from omegaconf import DictConfig + +from pipelinerl.async_llm import llm_async_generate +from pipelinerl.rollouts import RolloutResult, make_training_text +from tapeagents.core import Prompt +from tapeagents.llms.trainable import TrainableLLM + + +async def generate_counting_rollout( + cfg: DictConfig, + llm: TrainableLLM, + problem: dict, + session: aiohttp.ClientSession, +) -> RolloutResult: + letter = problem["letter"] + word = problem["word"] + count = problem["count"] + messages = [ + { + "role": "system", + "content": "You are a helpful assistant", + }, + { + "role": "user", + "content": f"How many {letter}'s are there in the word '{word}'? You can think step by step. Output the answer as number.", + } + ] + time_start = time.time() + llm_call = await llm_async_generate(llm, Prompt(messages=messages), session) + latency = time.time() - time_start + output_text = llm_call.output.content + + reward = 0 + error = 0 + if output_text: + answer = re.search("(\d+)", output_text) + if answer: + answer = int(answer.group(1)) + if answer == count: + reward = 1 + else: + error = 1 + else: + error = 1 + + training_text = make_training_text(llm, llm_call) + training_text.reward = reward + + finished = 1 if training_text.input_ids[-1] == llm.tokenizer.eos_token_id else 0 + + metrics = { + "reward": reward, + "success": reward, + "no_error": not error, + "no_answer": error, + "prompt_tokens": llm_call.prompt_length_tokens, + "output_tokens": llm_call.output_length_tokens, + "overflow": 0 if finished else 1, + } + + return RolloutResult( + training_texts=[training_text], + metrics=metrics, + latency=latency, + dataset_name=problem["dataset"] + ) + + +def load_problems(dataset_names: list[str]): + dir_path = os.path.dirname(os.path.realpath(__file__)) + problems = [] + for name in dataset_names: + file_path = os.path.join(dir_path, f"{name}.json") + if not os.path.exists(file_path): + raise FileNotFoundError(f"Dataset file {file_path} does not exist.") + with open(file_path, "r") as f: + dataset = json.load(f) + if not isinstance(dataset, list): + raise ValueError(f"Dataset {name} should be a list of problems.") + for problem in dataset: + if not isinstance(problem, dict) or "letter" not in problem or "word" not in problem or "count" not in problem: + raise ValueError(f"Problem {problem} in dataset {name} is invalid.") + problem["dataset"] = name + problems.append(problem) + return problems + diff --git a/pipelinerl/counting/test_counting_problems.json b/pipelinerl/counting/test_counting_problems.json new file mode 100644 index 0000000..0be3ade --- /dev/null +++ b/pipelinerl/counting/test_counting_problems.json @@ -0,0 +1,5002 @@ +[ + { + "word": "reinvented", + "letter": "i", + "count": 1 + }, + { + "word": "perinephrium", + "letter": "i", + "count": 2 + }, + { + "word": "life-rendering", + "letter": "e", + "count": 3 + }, + { + "word": "weather-glass", + "letter": "t", + "count": 1 + }, + { + "word": "spiny-skinned", + "letter": "s", + "count": 2 + }, + { + "word": "invigoratively", + "letter": "i", + "count": 3 + }, + { + "word": "malignancy", + "letter": "y", + "count": 1 + }, + { + "word": "lagena", + "letter": "a", + "count": 2 + }, + { + "word": "Jezebel", + "letter": "e", + "count": 3 + }, + { + "word": "baggily", + "letter": "b", + "count": 1 + }, + { + "word": "unfortune", + "letter": "n", + "count": 2 + }, + { + "word": "carte-de-visite", + "letter": "e", + "count": 3 + }, + { + "word": "colpitis", + "letter": "p", + "count": 1 + }, + { + "word": "empty-vaulted", + "letter": "e", + "count": 2 + }, + { + "word": "stetting", + "letter": "t", + "count": 3 + }, + { + "word": "bonnocks", + "letter": "b", + "count": 1 + }, + { + "word": "Waadt", + "letter": "a", + "count": 2 + }, + { + "word": "interveining", + "letter": "n", + "count": 3 + }, + { + "word": "Rockhampton", + "letter": "c", + "count": 1 + }, + { + "word": "Pro-arabic", + "letter": "a", + "count": 2 + }, + { + "word": "Walshville", + "letter": "l", + "count": 3 + }, + { + "word": "finish-cut", + "letter": "c", + "count": 1 + }, + { + "word": "redemptionless", + "letter": "s", + "count": 2 + }, + { + "word": "peripeties", + "letter": "e", + "count": 3 + }, + { + "word": "Seney", + "letter": "y", + "count": 1 + }, + { + "word": "bitterwood", + "letter": "o", + "count": 2 + }, + { + "word": "antecedental", + "letter": "e", + "count": 3 + }, + { + "word": "Seaforth", + "letter": "f", + "count": 1 + }, + { + "word": "Gasconism", + "letter": "s", + "count": 2 + }, + { + "word": "anachronically", + "letter": "a", + "count": 3 + }, + { + "word": "quirked", + "letter": "k", + "count": 1 + }, + { + "word": "deep-buried", + "letter": "d", + "count": 2 + }, + { + "word": "catacylsmic", + "letter": "c", + "count": 3 + }, + { + "word": "cultuses", + "letter": "c", + "count": 1 + }, + { + "word": "sudsman", + "letter": "s", + "count": 2 + }, + { + "word": "roturiers", + "letter": "r", + "count": 3 + }, + { + "word": "chaffeur-ship", + "letter": "a", + "count": 1 + }, + { + "word": "sound-producing", + "letter": "u", + "count": 2 + }, + { + "word": "lalling", + "letter": "l", + "count": 3 + }, + { + "word": "outwishing", + "letter": "n", + "count": 1 + }, + { + "word": "jacchus", + "letter": "c", + "count": 2 + }, + { + "word": "calpolli", + "letter": "l", + "count": 3 + }, + { + "word": "wowwows", + "letter": "s", + "count": 1 + }, + { + "word": "idium", + "letter": "i", + "count": 2 + }, + { + "word": "nonnaturalist", + "letter": "n", + "count": 3 + }, + { + "word": "erupt", + "letter": "r", + "count": 1 + }, + { + "word": "funereally", + "letter": "e", + "count": 2 + }, + { + "word": "neurohormonal", + "letter": "o", + "count": 3 + }, + { + "word": "radionuclide", + "letter": "e", + "count": 1 + }, + { + "word": "kozo", + "letter": "o", + "count": 2 + }, + { + "word": "azoxynaphthalene", + "letter": "a", + "count": 3 + }, + { + "word": "unsoiling", + "letter": "l", + "count": 1 + }, + { + "word": "sandalwort", + "letter": "a", + "count": 2 + }, + { + "word": "progressor", + "letter": "r", + "count": 3 + }, + { + "word": "dolorimetric", + "letter": "t", + "count": 1 + }, + { + "word": "awheel", + "letter": "e", + "count": 2 + }, + { + "word": "paragenetically", + "letter": "a", + "count": 3 + }, + { + "word": "premethodical", + "letter": "t", + "count": 1 + }, + { + "word": "hand-play", + "letter": "a", + "count": 2 + }, + { + "word": "semisocialistically", + "letter": "s", + "count": 3 + }, + { + "word": "on-hit", + "letter": "h", + "count": 1 + }, + { + "word": "isomere", + "letter": "e", + "count": 2 + }, + { + "word": "nonmanufacture", + "letter": "n", + "count": 3 + }, + { + "word": "standardbred", + "letter": "e", + "count": 1 + }, + { + "word": "high-hung", + "letter": "g", + "count": 2 + }, + { + "word": "yajnopavita", + "letter": "a", + "count": 3 + }, + { + "word": "Plainfield", + "letter": "n", + "count": 1 + }, + { + "word": "depthometer", + "letter": "t", + "count": 2 + }, + { + "word": "overlooked", + "letter": "o", + "count": 3 + }, + { + "word": "singletons", + "letter": "i", + "count": 1 + }, + { + "word": "juristically", + "letter": "i", + "count": 2 + }, + { + "word": "interdigitally", + "letter": "i", + "count": 3 + }, + { + "word": "self-hitting", + "letter": "s", + "count": 1 + }, + { + "word": "microprint", + "letter": "r", + "count": 2 + }, + { + "word": "inenarrability", + "letter": "i", + "count": 3 + }, + { + "word": "beveil", + "letter": "v", + "count": 1 + }, + { + "word": "ichthy-", + "letter": "h", + "count": 2 + }, + { + "word": "cataplane", + "letter": "a", + "count": 3 + }, + { + "word": "lipid", + "letter": "d", + "count": 1 + }, + { + "word": "demonologer", + "letter": "e", + "count": 2 + }, + { + "word": "disestablishment", + "letter": "s", + "count": 3 + }, + { + "word": "shoddy", + "letter": "s", + "count": 1 + }, + { + "word": "Cyrillian", + "letter": "l", + "count": 2 + }, + { + "word": "portentosity", + "letter": "t", + "count": 3 + }, + { + "word": "world-reviving", + "letter": "n", + "count": 1 + }, + { + "word": "overexertedly", + "letter": "r", + "count": 2 + }, + { + "word": "overpositiveness", + "letter": "e", + "count": 3 + }, + { + "word": "unstormed", + "letter": "t", + "count": 1 + }, + { + "word": "hydrosilicate", + "letter": "i", + "count": 2 + }, + { + "word": "transmaterial", + "letter": "a", + "count": 3 + }, + { + "word": "dismasting", + "letter": "d", + "count": 1 + }, + { + "word": "river-sundered", + "letter": "d", + "count": 2 + }, + { + "word": "dramatisable", + "letter": "a", + "count": 3 + }, + { + "word": "copens", + "letter": "s", + "count": 1 + }, + { + "word": "laparocele", + "letter": "l", + "count": 2 + }, + { + "word": "pretanning", + "letter": "n", + "count": 3 + }, + { + "word": "retunded", + "letter": "r", + "count": 1 + }, + { + "word": "nonconfiscatory", + "letter": "c", + "count": 2 + }, + { + "word": "moorworts", + "letter": "o", + "count": 3 + }, + { + "word": "wicked-looking", + "letter": "c", + "count": 1 + }, + { + "word": "appleringie", + "letter": "i", + "count": 2 + }, + { + "word": "Vladamar", + "letter": "a", + "count": 3 + }, + { + "word": "realer", + "letter": "a", + "count": 1 + }, + { + "word": "pluriflagellate", + "letter": "e", + "count": 2 + }, + { + "word": "exercises", + "letter": "e", + "count": 3 + }, + { + "word": "statuses", + "letter": "e", + "count": 1 + }, + { + "word": "dissympathy", + "letter": "s", + "count": 2 + }, + { + "word": "untolerableness", + "letter": "e", + "count": 3 + }, + { + "word": "co-ordinancy", + "letter": "a", + "count": 1 + }, + { + "word": "Leonanie", + "letter": "e", + "count": 2 + }, + { + "word": "overinsistencies", + "letter": "i", + "count": 3 + }, + { + "word": "nonpalatably", + "letter": "p", + "count": 1 + }, + { + "word": "vising", + "letter": "i", + "count": 2 + }, + { + "word": "antinaturalism", + "letter": "a", + "count": 3 + }, + { + "word": "pumicate", + "letter": "a", + "count": 1 + }, + { + "word": "recidive", + "letter": "i", + "count": 2 + }, + { + "word": "Chandlerville", + "letter": "l", + "count": 3 + }, + { + "word": "Minetto", + "letter": "M", + "count": 1 + }, + { + "word": "sircar", + "letter": "r", + "count": 2 + }, + { + "word": "superfetate", + "letter": "e", + "count": 3 + }, + { + "word": "drupe", + "letter": "r", + "count": 1 + }, + { + "word": "M.Ed.", + "letter": ".", + "count": 2 + }, + { + "word": "Cacana", + "letter": "a", + "count": 3 + }, + { + "word": "Kintyre", + "letter": "e", + "count": 1 + }, + { + "word": "cenotaphic", + "letter": "c", + "count": 2 + }, + { + "word": "marsupialization", + "letter": "i", + "count": 3 + }, + { + "word": "Fany", + "letter": "y", + "count": 1 + }, + { + "word": "mar-hawk", + "letter": "a", + "count": 2 + }, + { + "word": "ribroaster", + "letter": "r", + "count": 3 + }, + { + "word": "crocheting", + "letter": "t", + "count": 1 + }, + { + "word": "avengeress", + "letter": "s", + "count": 2 + }, + { + "word": "nonbranded", + "letter": "n", + "count": 3 + }, + { + "word": "amalgamations", + "letter": "g", + "count": 1 + }, + { + "word": "fascili", + "letter": "i", + "count": 2 + }, + { + "word": "homoeoblastic", + "letter": "o", + "count": 3 + }, + { + "word": "Chaplin", + "letter": "l", + "count": 1 + }, + { + "word": "town-dwelling", + "letter": "n", + "count": 2 + }, + { + "word": "slimpsiest", + "letter": "s", + "count": 3 + }, + { + "word": "ozotype", + "letter": "z", + "count": 1 + }, + { + "word": "repermission", + "letter": "s", + "count": 2 + }, + { + "word": "shiverweed", + "letter": "e", + "count": 3 + }, + { + "word": "gourounut", + "letter": "t", + "count": 1 + }, + { + "word": "multicorneal", + "letter": "l", + "count": 2 + }, + { + "word": "costless", + "letter": "s", + "count": 3 + }, + { + "word": "pibal", + "letter": "b", + "count": 1 + }, + { + "word": "prothoracic", + "letter": "o", + "count": 2 + }, + { + "word": "resplendently", + "letter": "e", + "count": 3 + }, + { + "word": "Xenurus", + "letter": "r", + "count": 1 + }, + { + "word": "coteline", + "letter": "e", + "count": 2 + }, + { + "word": "hederiferous", + "letter": "e", + "count": 3 + }, + { + "word": "boloed", + "letter": "l", + "count": 1 + }, + { + "word": "rehook", + "letter": "o", + "count": 2 + }, + { + "word": "neurocele", + "letter": "e", + "count": 3 + }, + { + "word": "Xylina", + "letter": "y", + "count": 1 + }, + { + "word": "imparadised", + "letter": "d", + "count": 2 + }, + { + "word": "regulation-proof", + "letter": "o", + "count": 3 + }, + { + "word": "epicarps", + "letter": "i", + "count": 1 + }, + { + "word": "taxeopody", + "letter": "o", + "count": 2 + }, + { + "word": "unwarrantableness", + "letter": "a", + "count": 3 + }, + { + "word": "include", + "letter": "i", + "count": 1 + }, + { + "word": "intracistern", + "letter": "i", + "count": 2 + }, + { + "word": "lallands", + "letter": "l", + "count": 3 + }, + { + "word": "ragamuffin", + "letter": "r", + "count": 1 + }, + { + "word": "otorhinolaryngology", + "letter": "r", + "count": 2 + }, + { + "word": "serriform", + "letter": "r", + "count": 3 + }, + { + "word": "quinonize", + "letter": "z", + "count": 1 + }, + { + "word": "redigitalize", + "letter": "e", + "count": 2 + }, + { + "word": "noninclusive", + "letter": "n", + "count": 3 + }, + { + "word": "dehorns", + "letter": "n", + "count": 1 + }, + { + "word": "benign", + "letter": "n", + "count": 2 + }, + { + "word": "speelless", + "letter": "e", + "count": 3 + }, + { + "word": "disattune", + "letter": "a", + "count": 1 + }, + { + "word": "left-handedness", + "letter": "s", + "count": 2 + }, + { + "word": "berede", + "letter": "e", + "count": 3 + }, + { + "word": "Dactylis", + "letter": "y", + "count": 1 + }, + { + "word": "nonsummons", + "letter": "m", + "count": 2 + }, + { + "word": "meddlesome", + "letter": "e", + "count": 3 + }, + { + "word": "wrings", + "letter": "r", + "count": 1 + }, + { + "word": "immortally", + "letter": "l", + "count": 2 + }, + { + "word": "bluisness", + "letter": "s", + "count": 3 + }, + { + "word": "representationalist", + "letter": "p", + "count": 1 + }, + { + "word": "crossroad", + "letter": "s", + "count": 2 + }, + { + "word": "wrong-endedness", + "letter": "n", + "count": 3 + }, + { + "word": "gurgly", + "letter": "r", + "count": 1 + }, + { + "word": "sloyds", + "letter": "s", + "count": 2 + }, + { + "word": "antimedication", + "letter": "i", + "count": 3 + }, + { + "word": "pignolia", + "letter": "l", + "count": 1 + }, + { + "word": "incandescency", + "letter": "e", + "count": 2 + }, + { + "word": "plebeianise", + "letter": "e", + "count": 3 + }, + { + "word": "prow's", + "letter": "o", + "count": 1 + }, + { + "word": "grassland", + "letter": "a", + "count": 2 + }, + { + "word": "swooses", + "letter": "s", + "count": 3 + }, + { + "word": "glandered", + "letter": "l", + "count": 1 + }, + { + "word": "sprenge", + "letter": "e", + "count": 2 + }, + { + "word": "sheepshearing", + "letter": "e", + "count": 3 + }, + { + "word": "syphilous", + "letter": "i", + "count": 1 + }, + { + "word": "malleiform", + "letter": "l", + "count": 2 + }, + { + "word": "hentriacontane", + "letter": "n", + "count": 3 + }, + { + "word": "capacitated", + "letter": "i", + "count": 1 + }, + { + "word": "downsizing", + "letter": "i", + "count": 2 + }, + { + "word": "bachelors-at-arms", + "letter": "a", + "count": 3 + }, + { + "word": "mutest", + "letter": "m", + "count": 1 + }, + { + "word": "unthematically", + "letter": "l", + "count": 2 + }, + { + "word": "food-producing", + "letter": "o", + "count": 3 + }, + { + "word": "lay-down", + "letter": "l", + "count": 1 + }, + { + "word": "perroquet", + "letter": "e", + "count": 2 + }, + { + "word": "ruddy-colored", + "letter": "d", + "count": 3 + }, + { + "word": "muslins", + "letter": "m", + "count": 1 + }, + { + "word": "bakal", + "letter": "a", + "count": 2 + }, + { + "word": "semicantilever", + "letter": "e", + "count": 3 + }, + { + "word": "kabobs", + "letter": "a", + "count": 1 + }, + { + "word": "tight-reining", + "letter": "g", + "count": 2 + }, + { + "word": "subscapularis", + "letter": "s", + "count": 3 + }, + { + "word": "Entomophthorales", + "letter": "E", + "count": 1 + }, + { + "word": "jageer", + "letter": "e", + "count": 2 + }, + { + "word": "scapuloradial", + "letter": "a", + "count": 3 + }, + { + "word": "Grafton", + "letter": "f", + "count": 1 + }, + { + "word": "plumipeds", + "letter": "p", + "count": 2 + }, + { + "word": "Seabee", + "letter": "e", + "count": 3 + }, + { + "word": "maladjustments", + "letter": "j", + "count": 1 + }, + { + "word": "plesiosauroid", + "letter": "s", + "count": 2 + }, + { + "word": "cerithioid", + "letter": "i", + "count": 3 + }, + { + "word": "hyponasty", + "letter": "p", + "count": 1 + }, + { + "word": "thiosinamine", + "letter": "n", + "count": 2 + }, + { + "word": "internuncios", + "letter": "n", + "count": 3 + }, + { + "word": "leucocythemia", + "letter": "h", + "count": 1 + }, + { + "word": "germ-forming", + "letter": "r", + "count": 2 + }, + { + "word": "beerhouse", + "letter": "e", + "count": 3 + }, + { + "word": "blue-bellied", + "letter": "u", + "count": 1 + }, + { + "word": "phrenologists", + "letter": "s", + "count": 2 + }, + { + "word": "penthouselike", + "letter": "e", + "count": 3 + }, + { + "word": "auchlet", + "letter": "h", + "count": 1 + }, + { + "word": "Scheck", + "letter": "c", + "count": 2 + }, + { + "word": "oenochoae", + "letter": "o", + "count": 3 + }, + { + "word": "legislations", + "letter": "e", + "count": 1 + }, + { + "word": "strangulate", + "letter": "a", + "count": 2 + }, + { + "word": "Palaeocene", + "letter": "e", + "count": 3 + }, + { + "word": "hagdown", + "letter": "w", + "count": 1 + }, + { + "word": "zoons", + "letter": "o", + "count": 2 + }, + { + "word": "Plataean", + "letter": "a", + "count": 3 + }, + { + "word": "dispensing", + "letter": "e", + "count": 1 + }, + { + "word": "rotten-minded", + "letter": "t", + "count": 2 + }, + { + "word": "re-rejoinder", + "letter": "e", + "count": 3 + }, + { + "word": "Venterea", + "letter": "n", + "count": 1 + }, + { + "word": "diluendo", + "letter": "d", + "count": 2 + }, + { + "word": "berserker", + "letter": "r", + "count": 3 + }, + { + "word": "counterresponse", + "letter": "u", + "count": 1 + }, + { + "word": "preparatives", + "letter": "r", + "count": 2 + }, + { + "word": "gallerylike", + "letter": "l", + "count": 3 + }, + { + "word": "trimastigate", + "letter": "g", + "count": 1 + }, + { + "word": "quernstone", + "letter": "n", + "count": 2 + }, + { + "word": "nonclassification", + "letter": "n", + "count": 3 + }, + { + "word": "self-destruction", + "letter": "o", + "count": 1 + }, + { + "word": "elelments", + "letter": "l", + "count": 2 + }, + { + "word": "illuvial", + "letter": "l", + "count": 3 + }, + { + "word": "Bascio", + "letter": "s", + "count": 1 + }, + { + "word": "quasi-essentially", + "letter": "a", + "count": 2 + }, + { + "word": "pelletlike", + "letter": "e", + "count": 3 + }, + { + "word": "protomagnate", + "letter": "g", + "count": 1 + }, + { + "word": "sorrow-bound", + "letter": "r", + "count": 2 + }, + { + "word": "photoesthetic", + "letter": "t", + "count": 3 + }, + { + "word": "spreads", + "letter": "a", + "count": 1 + }, + { + "word": "gendarmes", + "letter": "e", + "count": 2 + }, + { + "word": "bromomenorrhea", + "letter": "o", + "count": 3 + }, + { + "word": "diander", + "letter": "a", + "count": 1 + }, + { + "word": "peridiola", + "letter": "i", + "count": 2 + }, + { + "word": "reexpresses", + "letter": "s", + "count": 3 + }, + { + "word": "cinchonic", + "letter": "h", + "count": 1 + }, + { + "word": "macrophyllous", + "letter": "l", + "count": 2 + }, + { + "word": "mud-bespattered", + "letter": "e", + "count": 3 + }, + { + "word": "redeclares", + "letter": "d", + "count": 1 + }, + { + "word": "make-ready", + "letter": "e", + "count": 2 + }, + { + "word": "sievelike", + "letter": "e", + "count": 3 + }, + { + "word": "Harle", + "letter": "r", + "count": 1 + }, + { + "word": "light-bellied", + "letter": "e", + "count": 2 + }, + { + "word": "insensibleness", + "letter": "n", + "count": 3 + }, + { + "word": "nordenfelt", + "letter": "t", + "count": 1 + }, + { + "word": "wages-man", + "letter": "a", + "count": 2 + }, + { + "word": "purpurigenous", + "letter": "u", + "count": 3 + }, + { + "word": "minifying", + "letter": "g", + "count": 1 + }, + { + "word": "prefixedly", + "letter": "e", + "count": 2 + }, + { + "word": "dullification", + "letter": "i", + "count": 3 + }, + { + "word": "octuor", + "letter": "c", + "count": 1 + }, + { + "word": "nonpenetration", + "letter": "e", + "count": 2 + }, + { + "word": "taxaspidean", + "letter": "a", + "count": 3 + }, + { + "word": "purlman", + "letter": "r", + "count": 1 + }, + { + "word": "Plinyism", + "letter": "i", + "count": 2 + }, + { + "word": "noncounty", + "letter": "n", + "count": 3 + }, + { + "word": "nondeciduate", + "letter": "o", + "count": 1 + }, + { + "word": "pastils", + "letter": "s", + "count": 2 + }, + { + "word": "metatheses", + "letter": "e", + "count": 3 + }, + { + "word": "dirling", + "letter": "r", + "count": 1 + }, + { + "word": "jabberwockian", + "letter": "a", + "count": 2 + }, + { + "word": "puppysnatch", + "letter": "p", + "count": 3 + }, + { + "word": "uncorrectablely", + "letter": "t", + "count": 1 + }, + { + "word": "Buffalo", + "letter": "f", + "count": 2 + }, + { + "word": "half-sagittate", + "letter": "t", + "count": 3 + }, + { + "word": "prepositive", + "letter": "t", + "count": 1 + }, + { + "word": "murmurous", + "letter": "m", + "count": 2 + }, + { + "word": "consolidations", + "letter": "o", + "count": 3 + }, + { + "word": "mamboed", + "letter": "a", + "count": 1 + }, + { + "word": "homonymously", + "letter": "m", + "count": 2 + }, + { + "word": "detergents", + "letter": "e", + "count": 3 + }, + { + "word": "Ondrej", + "letter": "r", + "count": 1 + }, + { + "word": "navagium", + "letter": "a", + "count": 2 + }, + { + "word": "squirrel-headed", + "letter": "e", + "count": 3 + }, + { + "word": "laborously", + "letter": "a", + "count": 1 + }, + { + "word": "prothalloid", + "letter": "o", + "count": 2 + }, + { + "word": "pyosalpingitis", + "letter": "i", + "count": 3 + }, + { + "word": "re-erect", + "letter": "-", + "count": 1 + }, + { + "word": "taggers", + "letter": "g", + "count": 2 + }, + { + "word": "old-fogeydom", + "letter": "o", + "count": 3 + }, + { + "word": "Eubuleus", + "letter": "e", + "count": 1 + }, + { + "word": "slimmed", + "letter": "m", + "count": 2 + }, + { + "word": "unassistant", + "letter": "s", + "count": 3 + }, + { + "word": "MVY", + "letter": "V", + "count": 1 + }, + { + "word": "lawn-tractor", + "letter": "t", + "count": 2 + }, + { + "word": "twelve-tone", + "letter": "e", + "count": 3 + }, + { + "word": "Aduwa", + "letter": "a", + "count": 1 + }, + { + "word": "trappean", + "letter": "a", + "count": 2 + }, + { + "word": "ameerate", + "letter": "e", + "count": 3 + }, + { + "word": "infeof", + "letter": "n", + "count": 1 + }, + { + "word": "mischances", + "letter": "c", + "count": 2 + }, + { + "word": "overcommercialization", + "letter": "i", + "count": 3 + }, + { + "word": "iron-jawed", + "letter": "-", + "count": 1 + }, + { + "word": "quittances", + "letter": "t", + "count": 2 + }, + { + "word": "Carduaceae", + "letter": "a", + "count": 3 + }, + { + "word": "neophilologist", + "letter": "s", + "count": 1 + }, + { + "word": "convertingness", + "letter": "e", + "count": 2 + }, + { + "word": "pleurocele", + "letter": "e", + "count": 3 + }, + { + "word": "decolorizing", + "letter": "g", + "count": 1 + }, + { + "word": "senores", + "letter": "s", + "count": 2 + }, + { + "word": "aeolharmonica", + "letter": "a", + "count": 3 + }, + { + "word": "abduces", + "letter": "e", + "count": 1 + }, + { + "word": "horograph", + "letter": "r", + "count": 2 + }, + { + "word": "distended", + "letter": "d", + "count": 3 + }, + { + "word": "waterway's", + "letter": "y", + "count": 1 + }, + { + "word": "unchildishness", + "letter": "h", + "count": 2 + }, + { + "word": "presumptuous", + "letter": "u", + "count": 3 + }, + { + "word": "uncoloredly", + "letter": "u", + "count": 1 + }, + { + "word": "quarter-vine", + "letter": "r", + "count": 2 + }, + { + "word": "recidivist", + "letter": "i", + "count": 3 + }, + { + "word": "skirmishing", + "letter": "r", + "count": 1 + }, + { + "word": "shoneens", + "letter": "s", + "count": 2 + }, + { + "word": "Oryssus", + "letter": "s", + "count": 3 + }, + { + "word": "shrived", + "letter": "i", + "count": 1 + }, + { + "word": "sarcastic", + "letter": "a", + "count": 2 + }, + { + "word": "hypermonosyllable", + "letter": "l", + "count": 3 + }, + { + "word": "superannated", + "letter": "s", + "count": 1 + }, + { + "word": "Ciceronianism", + "letter": "n", + "count": 2 + }, + { + "word": "federalize", + "letter": "e", + "count": 3 + }, + { + "word": "strawberry-raspberry", + "letter": "w", + "count": 1 + }, + { + "word": "disrespective", + "letter": "i", + "count": 2 + }, + { + "word": "oversewed", + "letter": "e", + "count": 3 + }, + { + "word": "Michael", + "letter": "M", + "count": 1 + }, + { + "word": "resurrects", + "letter": "s", + "count": 2 + }, + { + "word": "acrosporous", + "letter": "o", + "count": 3 + }, + { + "word": "arthrozoic", + "letter": "a", + "count": 1 + }, + { + "word": "outsnore", + "letter": "o", + "count": 2 + }, + { + "word": "pseudolegislative", + "letter": "e", + "count": 3 + }, + { + "word": "antipredeterminant", + "letter": "d", + "count": 1 + }, + { + "word": "thyroprivous", + "letter": "o", + "count": 2 + }, + { + "word": "prediscouragement", + "letter": "e", + "count": 3 + }, + { + "word": "nonmanipulatory", + "letter": "r", + "count": 1 + }, + { + "word": "Redkey", + "letter": "e", + "count": 2 + }, + { + "word": "demonesses", + "letter": "e", + "count": 3 + }, + { + "word": "institutress", + "letter": "r", + "count": 1 + }, + { + "word": "Woolwa", + "letter": "o", + "count": 2 + }, + { + "word": "cynarctomachy", + "letter": "c", + "count": 3 + }, + { + "word": "FDIC", + "letter": "I", + "count": 1 + }, + { + "word": "befoulment", + "letter": "e", + "count": 2 + }, + { + "word": "dolomitization", + "letter": "o", + "count": 3 + }, + { + "word": "mechanotherapies", + "letter": "s", + "count": 1 + }, + { + "word": "permittance", + "letter": "e", + "count": 2 + }, + { + "word": "undermeaning", + "letter": "n", + "count": 3 + }, + { + "word": "nonpsychologic", + "letter": "s", + "count": 1 + }, + { + "word": "glycocholate", + "letter": "c", + "count": 2 + }, + { + "word": "injunction's", + "letter": "n", + "count": 3 + }, + { + "word": "corradial", + "letter": "d", + "count": 1 + }, + { + "word": "retastes", + "letter": "e", + "count": 2 + }, + { + "word": "pseudoparallel", + "letter": "l", + "count": 3 + }, + { + "word": "Procris", + "letter": "i", + "count": 1 + }, + { + "word": "scuddick", + "letter": "c", + "count": 2 + }, + { + "word": "calcaneocuboid", + "letter": "c", + "count": 3 + }, + { + "word": "trochilidae", + "letter": "c", + "count": 1 + }, + { + "word": "nationalness", + "letter": "s", + "count": 2 + }, + { + "word": "telegraphophone", + "letter": "e", + "count": 3 + }, + { + "word": "corseting", + "letter": "o", + "count": 1 + }, + { + "word": "rhinocerotic", + "letter": "o", + "count": 2 + }, + { + "word": "protozoans", + "letter": "o", + "count": 3 + }, + { + "word": "warm-clad", + "letter": "-", + "count": 1 + }, + { + "word": "natuary", + "letter": "a", + "count": 2 + }, + { + "word": "unsanguineously", + "letter": "n", + "count": 3 + }, + { + "word": "Suevian", + "letter": "a", + "count": 1 + }, + { + "word": "well-prized", + "letter": "e", + "count": 2 + }, + { + "word": "octoalloy", + "letter": "o", + "count": 3 + }, + { + "word": "fiery-cross", + "letter": "e", + "count": 1 + }, + { + "word": "currentwise", + "letter": "r", + "count": 2 + }, + { + "word": "noncotyledonous", + "letter": "n", + "count": 3 + }, + { + "word": "anarchism", + "letter": "h", + "count": 1 + }, + { + "word": "unobedience", + "letter": "n", + "count": 2 + }, + { + "word": "dermato-autoplasty", + "letter": "a", + "count": 3 + }, + { + "word": "neurochemistry", + "letter": "n", + "count": 1 + }, + { + "word": "dentiroster", + "letter": "r", + "count": 2 + }, + { + "word": "unremembrance", + "letter": "e", + "count": 3 + }, + { + "word": "Columelliaceae", + "letter": "o", + "count": 1 + }, + { + "word": "bevillain", + "letter": "i", + "count": 2 + }, + { + "word": "tree-runner", + "letter": "e", + "count": 3 + }, + { + "word": "inocyte", + "letter": "t", + "count": 1 + }, + { + "word": "stimulability", + "letter": "l", + "count": 2 + }, + { + "word": "mithridatise", + "letter": "i", + "count": 3 + }, + { + "word": "Damas", + "letter": "m", + "count": 1 + }, + { + "word": "nonblindingly", + "letter": "l", + "count": 2 + }, + { + "word": "palatization", + "letter": "a", + "count": 3 + }, + { + "word": "Heterodonta", + "letter": "r", + "count": 1 + }, + { + "word": "true-souled", + "letter": "u", + "count": 2 + }, + { + "word": "plutological", + "letter": "l", + "count": 3 + }, + { + "word": "risibles", + "letter": "l", + "count": 1 + }, + { + "word": "round-barreled", + "letter": "e", + "count": 2 + }, + { + "word": "Islamorada", + "letter": "a", + "count": 3 + }, + { + "word": "ill-seasoned", + "letter": "i", + "count": 1 + }, + { + "word": "synonymist", + "letter": "n", + "count": 2 + }, + { + "word": "bonheurs-du-jour", + "letter": "u", + "count": 3 + }, + { + "word": "supertrain", + "letter": "e", + "count": 1 + }, + { + "word": "repostponed", + "letter": "o", + "count": 2 + }, + { + "word": "anticonventionalism", + "letter": "i", + "count": 3 + }, + { + "word": "Manicamp", + "letter": "n", + "count": 1 + }, + { + "word": "tentillum", + "letter": "l", + "count": 2 + }, + { + "word": "nonexpanded", + "letter": "n", + "count": 3 + }, + { + "word": "tanzy", + "letter": "t", + "count": 1 + }, + { + "word": "succulences", + "letter": "u", + "count": 2 + }, + { + "word": "calcific", + "letter": "c", + "count": 3 + }, + { + "word": "urceoli", + "letter": "i", + "count": 1 + }, + { + "word": "internobasal", + "letter": "n", + "count": 2 + }, + { + "word": "uncovetousness", + "letter": "s", + "count": 3 + }, + { + "word": "protuberancies", + "letter": "o", + "count": 1 + }, + { + "word": "basilinna", + "letter": "n", + "count": 2 + }, + { + "word": "laterifloral", + "letter": "l", + "count": 3 + }, + { + "word": "stockholdings", + "letter": "c", + "count": 1 + }, + { + "word": "Leguminosae", + "letter": "e", + "count": 2 + }, + { + "word": "slumminess", + "letter": "s", + "count": 3 + }, + { + "word": "serviential", + "letter": "s", + "count": 1 + }, + { + "word": "conichalcite", + "letter": "i", + "count": 2 + }, + { + "word": "laudification", + "letter": "i", + "count": 3 + }, + { + "word": "Queneau", + "letter": "a", + "count": 1 + }, + { + "word": "Seema", + "letter": "e", + "count": 2 + }, + { + "word": "centerwise", + "letter": "e", + "count": 3 + }, + { + "word": "proportionate", + "letter": "a", + "count": 1 + }, + { + "word": "torpedomen", + "letter": "e", + "count": 2 + }, + { + "word": "high-climbing", + "letter": "i", + "count": 3 + }, + { + "word": "lenses", + "letter": "l", + "count": 1 + }, + { + "word": "Loogootee", + "letter": "e", + "count": 2 + }, + { + "word": "unsocialistic", + "letter": "i", + "count": 3 + }, + { + "word": "carapo", + "letter": "c", + "count": 1 + }, + { + "word": "anthropomancy", + "letter": "n", + "count": 2 + }, + { + "word": "congenitalness", + "letter": "n", + "count": 3 + }, + { + "word": "pterygosphenoid", + "letter": "d", + "count": 1 + }, + { + "word": "Bobadilla", + "letter": "a", + "count": 2 + }, + { + "word": "Moeritheriidae", + "letter": "i", + "count": 3 + }, + { + "word": "apocrenic", + "letter": "r", + "count": 1 + }, + { + "word": "seasonings", + "letter": "n", + "count": 2 + }, + { + "word": "unrejuvenating", + "letter": "n", + "count": 3 + }, + { + "word": "odontophore", + "letter": "p", + "count": 1 + }, + { + "word": "Soumaintrin", + "letter": "n", + "count": 2 + }, + { + "word": "polyonomy", + "letter": "o", + "count": 3 + }, + { + "word": "puromucous", + "letter": "c", + "count": 1 + }, + { + "word": "robalito", + "letter": "o", + "count": 2 + }, + { + "word": "Tamarah", + "letter": "a", + "count": 3 + }, + { + "word": "exosperm", + "letter": "r", + "count": 1 + }, + { + "word": "blatantly", + "letter": "t", + "count": 2 + }, + { + "word": "tetanomotor", + "letter": "t", + "count": 3 + }, + { + "word": "roofer", + "letter": "f", + "count": 1 + }, + { + "word": "parasubphonate", + "letter": "p", + "count": 2 + }, + { + "word": "disenfranchises", + "letter": "s", + "count": 3 + }, + { + "word": "rathe-ripe", + "letter": "p", + "count": 1 + }, + { + "word": "parencephalon", + "letter": "a", + "count": 2 + }, + { + "word": "perpetrator's", + "letter": "r", + "count": 3 + }, + { + "word": "quott", + "letter": "q", + "count": 1 + }, + { + "word": "foo", + "letter": "o", + "count": 2 + }, + { + "word": "bitterroot", + "letter": "t", + "count": 3 + }, + { + "word": "potch", + "letter": "p", + "count": 1 + }, + { + "word": "night-cheering", + "letter": "i", + "count": 2 + }, + { + "word": "sallow-cheeked", + "letter": "e", + "count": 3 + }, + { + "word": "Rhacophorus", + "letter": "r", + "count": 1 + }, + { + "word": "overmerry", + "letter": "e", + "count": 2 + }, + { + "word": "nonaudibleness", + "letter": "n", + "count": 3 + }, + { + "word": "assertor", + "letter": "e", + "count": 1 + }, + { + "word": "museum's", + "letter": "m", + "count": 2 + }, + { + "word": "straddled", + "letter": "d", + "count": 3 + }, + { + "word": "Cottenham", + "letter": "h", + "count": 1 + }, + { + "word": "derepression", + "letter": "s", + "count": 2 + }, + { + "word": "beweeping", + "letter": "e", + "count": 3 + }, + { + "word": "half-cracked", + "letter": "l", + "count": 1 + }, + { + "word": "Harriston", + "letter": "r", + "count": 2 + }, + { + "word": "recommitment", + "letter": "m", + "count": 3 + }, + { + "word": "Donald", + "letter": "o", + "count": 1 + }, + { + "word": "governable", + "letter": "e", + "count": 2 + }, + { + "word": "nongratifying", + "letter": "n", + "count": 3 + }, + { + "word": "self-mortified", + "letter": "r", + "count": 1 + }, + { + "word": "undercoursing", + "letter": "n", + "count": 2 + }, + { + "word": "palaeopotamology", + "letter": "a", + "count": 3 + }, + { + "word": "autoplasty", + "letter": "u", + "count": 1 + }, + { + "word": "billabong", + "letter": "l", + "count": 2 + }, + { + "word": "dividends", + "letter": "d", + "count": 3 + }, + { + "word": "Madlyn", + "letter": "M", + "count": 1 + }, + { + "word": "Woodlyn", + "letter": "o", + "count": 2 + }, + { + "word": "better-balanced", + "letter": "e", + "count": 3 + }, + { + "word": "water-living", + "letter": "l", + "count": 1 + }, + { + "word": "disburdened", + "letter": "e", + "count": 2 + }, + { + "word": "enjoyableness", + "letter": "e", + "count": 3 + }, + { + "word": "hele", + "letter": "l", + "count": 1 + }, + { + "word": "decorticate", + "letter": "t", + "count": 2 + }, + { + "word": "nonrevealing", + "letter": "n", + "count": 3 + }, + { + "word": "property", + "letter": "y", + "count": 1 + }, + { + "word": "surnamers", + "letter": "s", + "count": 2 + }, + { + "word": "undiscreetness", + "letter": "s", + "count": 3 + }, + { + "word": "amphoriskos", + "letter": "r", + "count": 1 + }, + { + "word": "Dermobranchia", + "letter": "a", + "count": 2 + }, + { + "word": "valorousness", + "letter": "s", + "count": 3 + }, + { + "word": "Hobbsville", + "letter": "o", + "count": 1 + }, + { + "word": "desuetudes", + "letter": "u", + "count": 2 + }, + { + "word": "scrapiness", + "letter": "s", + "count": 3 + }, + { + "word": "intracranial", + "letter": "t", + "count": 1 + }, + { + "word": "Coelenterata", + "letter": "t", + "count": 2 + }, + { + "word": "Valladolid", + "letter": "l", + "count": 3 + }, + { + "word": "zygomorphy", + "letter": "p", + "count": 1 + }, + { + "word": "mastoiditis", + "letter": "s", + "count": 2 + }, + { + "word": "phlebolithiasis", + "letter": "i", + "count": 3 + }, + { + "word": "slideways", + "letter": "e", + "count": 1 + }, + { + "word": "Theresina", + "letter": "e", + "count": 2 + }, + { + "word": "wotteth", + "letter": "t", + "count": 3 + }, + { + "word": "gliffs", + "letter": "s", + "count": 1 + }, + { + "word": "decurrences", + "letter": "c", + "count": 2 + }, + { + "word": "sunny-natured", + "letter": "n", + "count": 3 + }, + { + "word": "sulphoparaldehyde", + "letter": "u", + "count": 1 + }, + { + "word": "dermutation", + "letter": "t", + "count": 2 + }, + { + "word": "consonantic", + "letter": "n", + "count": 3 + }, + { + "word": "beswink", + "letter": "s", + "count": 1 + }, + { + "word": "natterjack", + "letter": "a", + "count": 2 + }, + { + "word": "systilius", + "letter": "s", + "count": 3 + }, + { + "word": "lehrman", + "letter": "h", + "count": 1 + }, + { + "word": "nonsinging", + "letter": "i", + "count": 2 + }, + { + "word": "cholesterolemia", + "letter": "e", + "count": 3 + }, + { + "word": "Fernandez", + "letter": "d", + "count": 1 + }, + { + "word": "nonexactable", + "letter": "a", + "count": 2 + }, + { + "word": "papilionoid", + "letter": "i", + "count": 3 + }, + { + "word": "gas-plant", + "letter": "t", + "count": 1 + }, + { + "word": "inconvenienced", + "letter": "i", + "count": 2 + }, + { + "word": "thigging", + "letter": "g", + "count": 3 + }, + { + "word": "gymnogenous", + "letter": "m", + "count": 1 + }, + { + "word": "uncleanse", + "letter": "n", + "count": 2 + }, + { + "word": "ten-talented", + "letter": "e", + "count": 3 + }, + { + "word": "sulphantimonial", + "letter": "u", + "count": 1 + }, + { + "word": "broad-wheeled", + "letter": "d", + "count": 2 + }, + { + "word": "pinniwinkis", + "letter": "n", + "count": 3 + }, + { + "word": "Bianchini", + "letter": "B", + "count": 1 + }, + { + "word": "Peacham", + "letter": "a", + "count": 2 + }, + { + "word": "spooniness", + "letter": "s", + "count": 3 + }, + { + "word": "muscaris", + "letter": "c", + "count": 1 + }, + { + "word": "niyanda", + "letter": "a", + "count": 2 + }, + { + "word": "oxyacetylene", + "letter": "e", + "count": 3 + }, + { + "word": "comparative", + "letter": "r", + "count": 1 + }, + { + "word": "Dictyosiphonaceae", + "letter": "e", + "count": 2 + }, + { + "word": "nonsensorial", + "letter": "n", + "count": 3 + }, + { + "word": "Heligmus", + "letter": "m", + "count": 1 + }, + { + "word": "cathection", + "letter": "t", + "count": 2 + }, + { + "word": "sayableness", + "letter": "s", + "count": 3 + }, + { + "word": "fishpotter", + "letter": "s", + "count": 1 + }, + { + "word": "laboringly", + "letter": "l", + "count": 2 + }, + { + "word": "intransitiveness", + "letter": "i", + "count": 3 + }, + { + "word": "salmonellae", + "letter": "o", + "count": 1 + }, + { + "word": "conserve", + "letter": "e", + "count": 2 + }, + { + "word": "world-swallowing", + "letter": "l", + "count": 3 + }, + { + "word": "homoeozoic", + "letter": "h", + "count": 1 + }, + { + "word": "Gavrielle", + "letter": "l", + "count": 2 + }, + { + "word": "peternet", + "letter": "e", + "count": 3 + }, + { + "word": "soapworts", + "letter": "t", + "count": 1 + }, + { + "word": "multitheism", + "letter": "m", + "count": 2 + }, + { + "word": "inorganization", + "letter": "n", + "count": 3 + }, + { + "word": "kangaroo-rat", + "letter": "t", + "count": 1 + }, + { + "word": "flummeries", + "letter": "e", + "count": 2 + }, + { + "word": "nonnebular", + "letter": "n", + "count": 3 + }, + { + "word": "self-improvable", + "letter": "p", + "count": 1 + }, + { + "word": "loosing", + "letter": "o", + "count": 2 + }, + { + "word": "oligosyllabic", + "letter": "l", + "count": 3 + }, + { + "word": "ungrappled", + "letter": "a", + "count": 1 + }, + { + "word": "barragon", + "letter": "a", + "count": 2 + }, + { + "word": "unsociological", + "letter": "o", + "count": 3 + }, + { + "word": "kick-off", + "letter": "i", + "count": 1 + }, + { + "word": "witchuck", + "letter": "c", + "count": 2 + }, + { + "word": "tenebrose", + "letter": "e", + "count": 3 + }, + { + "word": "aurochloride", + "letter": "h", + "count": 1 + }, + { + "word": "unsafest", + "letter": "s", + "count": 2 + }, + { + "word": "paraphilia", + "letter": "a", + "count": 3 + }, + { + "word": "eye-tooth", + "letter": "y", + "count": 1 + }, + { + "word": "nondiathermanous", + "letter": "o", + "count": 2 + }, + { + "word": "pronatoflexor", + "letter": "o", + "count": 3 + }, + { + "word": "psychrometer", + "letter": "y", + "count": 1 + }, + { + "word": "onrushing", + "letter": "n", + "count": 2 + }, + { + "word": "wakefulnesses", + "letter": "s", + "count": 3 + }, + { + "word": "coal-handling", + "letter": "i", + "count": 1 + }, + { + "word": "hefted", + "letter": "e", + "count": 2 + }, + { + "word": "superspinous", + "letter": "s", + "count": 3 + }, + { + "word": "Chinatown", + "letter": "h", + "count": 1 + }, + { + "word": "Hydrocorisae", + "letter": "r", + "count": 2 + }, + { + "word": "wittinesses", + "letter": "s", + "count": 3 + }, + { + "word": "discommons", + "letter": "n", + "count": 1 + }, + { + "word": "unbeatable", + "letter": "a", + "count": 2 + }, + { + "word": "seleniate", + "letter": "e", + "count": 3 + }, + { + "word": "noninherence", + "letter": "c", + "count": 1 + }, + { + "word": "overleapt", + "letter": "e", + "count": 2 + }, + { + "word": "liverishness", + "letter": "s", + "count": 3 + }, + { + "word": "umbellets", + "letter": "b", + "count": 1 + }, + { + "word": "Marianic", + "letter": "a", + "count": 2 + }, + { + "word": "bibliotherapies", + "letter": "i", + "count": 3 + }, + { + "word": "Eveleth", + "letter": "l", + "count": 1 + }, + { + "word": "technicological", + "letter": "i", + "count": 2 + }, + { + "word": "jovializing", + "letter": "i", + "count": 3 + }, + { + "word": "curricularization", + "letter": "z", + "count": 1 + }, + { + "word": "MacLean", + "letter": "a", + "count": 2 + }, + { + "word": "towelette", + "letter": "e", + "count": 3 + }, + { + "word": "indie", + "letter": "d", + "count": 1 + }, + { + "word": "lap-lap", + "letter": "a", + "count": 2 + }, + { + "word": "antiphilosophism", + "letter": "i", + "count": 3 + }, + { + "word": "nonsedentariness", + "letter": "d", + "count": 1 + }, + { + "word": "jamrosade", + "letter": "a", + "count": 2 + }, + { + "word": "Normalville", + "letter": "l", + "count": 3 + }, + { + "word": "outfrown", + "letter": "r", + "count": 1 + }, + { + "word": "crossed-h", + "letter": "s", + "count": 2 + }, + { + "word": "nonimbricative", + "letter": "i", + "count": 3 + }, + { + "word": "POS", + "letter": "O", + "count": 1 + }, + { + "word": "redefinition", + "letter": "n", + "count": 2 + }, + { + "word": "laboratorian", + "letter": "a", + "count": 3 + }, + { + "word": "stags", + "letter": "t", + "count": 1 + }, + { + "word": "whedder", + "letter": "d", + "count": 2 + }, + { + "word": "do-good", + "letter": "o", + "count": 3 + }, + { + "word": "earthset", + "letter": "r", + "count": 1 + }, + { + "word": "weakmouthed", + "letter": "e", + "count": 2 + }, + { + "word": "questionability", + "letter": "i", + "count": 3 + }, + { + "word": "hospitator", + "letter": "i", + "count": 1 + }, + { + "word": "Rockhall", + "letter": "l", + "count": 2 + }, + { + "word": "elementish", + "letter": "e", + "count": 3 + }, + { + "word": "turbescency", + "letter": "u", + "count": 1 + }, + { + "word": "hearthless", + "letter": "e", + "count": 2 + }, + { + "word": "pharmacologically", + "letter": "a", + "count": 3 + }, + { + "word": "Castoridae", + "letter": "s", + "count": 1 + }, + { + "word": "impetuous", + "letter": "u", + "count": 2 + }, + { + "word": "salutariness", + "letter": "s", + "count": 3 + }, + { + "word": "Acinetaria", + "letter": "n", + "count": 1 + }, + { + "word": "hypothecal", + "letter": "h", + "count": 2 + }, + { + "word": "stockingless", + "letter": "s", + "count": 3 + }, + { + "word": "insist", + "letter": "t", + "count": 1 + }, + { + "word": "single-seater", + "letter": "s", + "count": 2 + }, + { + "word": "naked-flowered", + "letter": "e", + "count": 3 + }, + { + "word": "zirconiums", + "letter": "u", + "count": 1 + }, + { + "word": "televises", + "letter": "s", + "count": 2 + }, + { + "word": "quinquennially", + "letter": "n", + "count": 3 + }, + { + "word": "downcastly", + "letter": "w", + "count": 1 + }, + { + "word": "Kikatsik", + "letter": "k", + "count": 2 + }, + { + "word": "sweet-chaste", + "letter": "e", + "count": 3 + }, + { + "word": "nonsymbolical", + "letter": "m", + "count": 1 + }, + { + "word": "twelfthly", + "letter": "t", + "count": 2 + }, + { + "word": "leatherine", + "letter": "e", + "count": 3 + }, + { + "word": "preabstract", + "letter": "c", + "count": 1 + }, + { + "word": "pebbleware", + "letter": "b", + "count": 2 + }, + { + "word": "aplacophorous", + "letter": "o", + "count": 3 + }, + { + "word": "dicephalus", + "letter": "c", + "count": 1 + }, + { + "word": "perchlor-", + "letter": "r", + "count": 2 + }, + { + "word": "pale-yellow", + "letter": "l", + "count": 3 + }, + { + "word": "alcoholism", + "letter": "h", + "count": 1 + }, + { + "word": "lherzolite", + "letter": "e", + "count": 2 + }, + { + "word": "baboodom", + "letter": "o", + "count": 3 + }, + { + "word": "otiatry", + "letter": "a", + "count": 1 + }, + { + "word": "slockingstone", + "letter": "s", + "count": 2 + }, + { + "word": "succulency", + "letter": "c", + "count": 3 + }, + { + "word": "facebow", + "letter": "w", + "count": 1 + }, + { + "word": "cyclonical", + "letter": "l", + "count": 2 + }, + { + "word": "sengreen", + "letter": "e", + "count": 3 + }, + { + "word": "Tice", + "letter": "i", + "count": 1 + }, + { + "word": "reafforestation", + "letter": "e", + "count": 2 + }, + { + "word": "thought-lighted", + "letter": "h", + "count": 3 + }, + { + "word": "Placitas", + "letter": "P", + "count": 1 + }, + { + "word": "squintier", + "letter": "i", + "count": 2 + }, + { + "word": "ill-favoredly", + "letter": "l", + "count": 3 + }, + { + "word": "Agib", + "letter": "i", + "count": 1 + }, + { + "word": "Staphylinoidea", + "letter": "i", + "count": 2 + }, + { + "word": "nonprescription", + "letter": "n", + "count": 3 + }, + { + "word": "reins", + "letter": "r", + "count": 1 + }, + { + "word": "inogenic", + "letter": "n", + "count": 2 + }, + { + "word": "digestibleness", + "letter": "s", + "count": 3 + }, + { + "word": "totipalmation", + "letter": "p", + "count": 1 + }, + { + "word": "minutia", + "letter": "i", + "count": 2 + }, + { + "word": "pseudolamellibranchiate", + "letter": "a", + "count": 3 + }, + { + "word": "qanats", + "letter": "n", + "count": 1 + }, + { + "word": "melitaemia", + "letter": "e", + "count": 2 + }, + { + "word": "insession", + "letter": "s", + "count": 3 + }, + { + "word": "paedogenic", + "letter": "g", + "count": 1 + }, + { + "word": "distortion", + "letter": "t", + "count": 2 + }, + { + "word": "ungrammaticism", + "letter": "m", + "count": 3 + }, + { + "word": "cephalostyle", + "letter": "t", + "count": 1 + }, + { + "word": "outmeasured", + "letter": "u", + "count": 2 + }, + { + "word": "unrepeatable", + "letter": "e", + "count": 3 + }, + { + "word": "beekeeper", + "letter": "p", + "count": 1 + }, + { + "word": "theorise", + "letter": "e", + "count": 2 + }, + { + "word": "bacteriophagia", + "letter": "a", + "count": 3 + }, + { + "word": "inconsequent", + "letter": "q", + "count": 1 + }, + { + "word": "ultramelancholy", + "letter": "a", + "count": 2 + }, + { + "word": "transfusions", + "letter": "s", + "count": 3 + }, + { + "word": "unroofing", + "letter": "u", + "count": 1 + }, + { + "word": "skeppund", + "letter": "p", + "count": 2 + }, + { + "word": "ricinic", + "letter": "i", + "count": 3 + }, + { + "word": "disally", + "letter": "a", + "count": 1 + }, + { + "word": "hectograph", + "letter": "h", + "count": 2 + }, + { + "word": "sciolistic", + "letter": "i", + "count": 3 + }, + { + "word": "shoebindery", + "letter": "y", + "count": 1 + }, + { + "word": "phi-phenomenon", + "letter": "h", + "count": 2 + }, + { + "word": "trichinize", + "letter": "i", + "count": 3 + }, + { + "word": "mulattoism", + "letter": "o", + "count": 1 + }, + { + "word": "peruke", + "letter": "e", + "count": 2 + }, + { + "word": "acrodactyla", + "letter": "a", + "count": 3 + }, + { + "word": "unperuked", + "letter": "p", + "count": 1 + }, + { + "word": "heartbreakingly", + "letter": "r", + "count": 2 + }, + { + "word": "institutress", + "letter": "t", + "count": 3 + }, + { + "word": "shamuses", + "letter": "h", + "count": 1 + }, + { + "word": "Occidentalism", + "letter": "i", + "count": 2 + }, + { + "word": "sociology", + "letter": "o", + "count": 3 + }, + { + "word": "operette", + "letter": "r", + "count": 1 + }, + { + "word": "finger-and-toe", + "letter": "-", + "count": 2 + }, + { + "word": "contingence", + "letter": "n", + "count": 3 + }, + { + "word": "sainthood", + "letter": "d", + "count": 1 + }, + { + "word": "melampyritol", + "letter": "l", + "count": 2 + }, + { + "word": "unintrudingly", + "letter": "n", + "count": 3 + }, + { + "word": "quartin", + "letter": "r", + "count": 1 + }, + { + "word": "celliferous", + "letter": "e", + "count": 2 + }, + { + "word": "jack-at-a-pinch", + "letter": "-", + "count": 3 + }, + { + "word": "Creswell", + "letter": "r", + "count": 1 + }, + { + "word": "anhistous", + "letter": "s", + "count": 2 + }, + { + "word": "releases", + "letter": "e", + "count": 3 + }, + { + "word": "nondualistic", + "letter": "o", + "count": 1 + }, + { + "word": "none-so-pretties", + "letter": "s", + "count": 2 + }, + { + "word": "actinobranchia", + "letter": "a", + "count": 3 + }, + { + "word": "hypsometry", + "letter": "e", + "count": 1 + }, + { + "word": "predictation", + "letter": "t", + "count": 2 + }, + { + "word": "heterocline", + "letter": "e", + "count": 3 + }, + { + "word": "fibrinoid", + "letter": "b", + "count": 1 + }, + { + "word": "mancipleship", + "letter": "p", + "count": 2 + }, + { + "word": "Poussinisme", + "letter": "s", + "count": 3 + }, + { + "word": "trame", + "letter": "r", + "count": 1 + }, + { + "word": "workweeks", + "letter": "w", + "count": 2 + }, + { + "word": "sunburning", + "letter": "n", + "count": 3 + }, + { + "word": "terraria", + "letter": "e", + "count": 1 + }, + { + "word": "unrivalling", + "letter": "i", + "count": 2 + }, + { + "word": "Laodamia", + "letter": "a", + "count": 3 + }, + { + "word": "Papst", + "letter": "p", + "count": 1 + }, + { + "word": "pumping", + "letter": "p", + "count": 2 + }, + { + "word": "antisensuously", + "letter": "s", + "count": 3 + }, + { + "word": "semithoroughfare", + "letter": "t", + "count": 1 + }, + { + "word": "vulcanizable", + "letter": "a", + "count": 2 + }, + { + "word": "Breathitt", + "letter": "t", + "count": 3 + }, + { + "word": "dickier", + "letter": "r", + "count": 1 + }, + { + "word": "unvalorousness", + "letter": "o", + "count": 2 + }, + { + "word": "carburator", + "letter": "r", + "count": 3 + }, + { + "word": "Teevens", + "letter": "v", + "count": 1 + }, + { + "word": "gormandizing", + "letter": "i", + "count": 2 + }, + { + "word": "yatagans", + "letter": "a", + "count": 3 + }, + { + "word": "tomb", + "letter": "m", + "count": 1 + }, + { + "word": "remainderman", + "letter": "r", + "count": 2 + }, + { + "word": "Gamopetalae", + "letter": "a", + "count": 3 + }, + { + "word": "eighteens", + "letter": "t", + "count": 1 + }, + { + "word": "gangbang", + "letter": "a", + "count": 2 + }, + { + "word": "capacitates", + "letter": "a", + "count": 3 + }, + { + "word": "honey-guide", + "letter": "-", + "count": 1 + }, + { + "word": "heartwood", + "letter": "o", + "count": 2 + }, + { + "word": "racialization", + "letter": "i", + "count": 3 + }, + { + "word": "photoluminescently", + "letter": "m", + "count": 1 + }, + { + "word": "infidelities", + "letter": "e", + "count": 2 + }, + { + "word": "Massachuset", + "letter": "s", + "count": 3 + }, + { + "word": "salmon-tinted", + "letter": "m", + "count": 1 + }, + { + "word": "three-thorned", + "letter": "t", + "count": 2 + }, + { + "word": "conceivability", + "letter": "i", + "count": 3 + }, + { + "word": "nonprincipled", + "letter": "e", + "count": 1 + }, + { + "word": "uncracked", + "letter": "c", + "count": 2 + }, + { + "word": "homomorphism's", + "letter": "m", + "count": 3 + }, + { + "word": "Cakavci", + "letter": "c", + "count": 1 + }, + { + "word": "coquelicot", + "letter": "c", + "count": 2 + }, + { + "word": "piscicapturist", + "letter": "i", + "count": 3 + }, + { + "word": "oblatio", + "letter": "b", + "count": 1 + }, + { + "word": "semanticist's", + "letter": "t", + "count": 2 + }, + { + "word": "overallegiance", + "letter": "e", + "count": 3 + }, + { + "word": "synderesis", + "letter": "d", + "count": 1 + }, + { + "word": "fissipalmation", + "letter": "s", + "count": 2 + }, + { + "word": "politico-social", + "letter": "i", + "count": 3 + }, + { + "word": "Withania", + "letter": "n", + "count": 1 + }, + { + "word": "connoted", + "letter": "n", + "count": 2 + }, + { + "word": "incomprehensibleness", + "letter": "s", + "count": 3 + }, + { + "word": "violatory", + "letter": "y", + "count": 1 + }, + { + "word": "retrain", + "letter": "r", + "count": 2 + }, + { + "word": "theocentricity", + "letter": "t", + "count": 3 + }, + { + "word": "atrocities", + "letter": "a", + "count": 1 + }, + { + "word": "reshoeing", + "letter": "e", + "count": 2 + }, + { + "word": "well-identified", + "letter": "e", + "count": 3 + }, + { + "word": "splice", + "letter": "e", + "count": 1 + }, + { + "word": "anisocytosis", + "letter": "i", + "count": 2 + }, + { + "word": "sedgelike", + "letter": "e", + "count": 3 + }, + { + "word": "prounionist", + "letter": "p", + "count": 1 + }, + { + "word": "tsks", + "letter": "s", + "count": 2 + }, + { + "word": "farsalah", + "letter": "a", + "count": 3 + }, + { + "word": "deracination", + "letter": "e", + "count": 1 + }, + { + "word": "reillustrate", + "letter": "t", + "count": 2 + }, + { + "word": "painkilling", + "letter": "i", + "count": 3 + }, + { + "word": "reindulged", + "letter": "l", + "count": 1 + }, + { + "word": "Yonkersite", + "letter": "e", + "count": 2 + }, + { + "word": "nominalistic", + "letter": "i", + "count": 3 + }, + { + "word": "intrabiontic", + "letter": "a", + "count": 1 + }, + { + "word": "ultroneousness", + "letter": "o", + "count": 2 + }, + { + "word": "heart-freezing", + "letter": "e", + "count": 3 + }, + { + "word": "dog's-eared", + "letter": "a", + "count": 1 + }, + { + "word": "mystagogue", + "letter": "g", + "count": 2 + }, + { + "word": "paleornithological", + "letter": "o", + "count": 3 + }, + { + "word": "promilitarist", + "letter": "o", + "count": 1 + }, + { + "word": "pseudoapologetically", + "letter": "e", + "count": 2 + }, + { + "word": "delustered", + "letter": "e", + "count": 3 + }, + { + "word": "mainprize", + "letter": "a", + "count": 1 + }, + { + "word": "rotundifolious", + "letter": "u", + "count": 2 + }, + { + "word": "oryzivorous", + "letter": "o", + "count": 3 + }, + { + "word": "chronicler", + "letter": "o", + "count": 1 + }, + { + "word": "contaminant", + "letter": "t", + "count": 2 + }, + { + "word": "daemonology", + "letter": "o", + "count": 3 + }, + { + "word": "hemoerythrin", + "letter": "y", + "count": 1 + }, + { + "word": "convolutedly", + "letter": "o", + "count": 2 + }, + { + "word": "pressureless", + "letter": "e", + "count": 3 + }, + { + "word": "underfeel", + "letter": "d", + "count": 1 + }, + { + "word": "eichwaldite", + "letter": "i", + "count": 2 + }, + { + "word": "self-gullery", + "letter": "l", + "count": 3 + }, + { + "word": "studiedly", + "letter": "l", + "count": 1 + }, + { + "word": "shiningness", + "letter": "i", + "count": 2 + }, + { + "word": "esophagomycosis", + "letter": "o", + "count": 3 + }, + { + "word": "hyperintense", + "letter": "i", + "count": 1 + }, + { + "word": "acetylenic", + "letter": "e", + "count": 2 + }, + { + "word": "Bessarabia", + "letter": "a", + "count": 3 + }, + { + "word": "ctenodactyl", + "letter": "y", + "count": 1 + }, + { + "word": "eau-de-vie", + "letter": "-", + "count": 2 + }, + { + "word": "unexceptionable", + "letter": "e", + "count": 3 + }, + { + "word": "Gullstrand", + "letter": "r", + "count": 1 + }, + { + "word": "odoriferously", + "letter": "r", + "count": 2 + }, + { + "word": "oversettled", + "letter": "e", + "count": 3 + }, + { + "word": "hymnography", + "letter": "m", + "count": 1 + }, + { + "word": "anhysteretic", + "letter": "t", + "count": 2 + }, + { + "word": "instantiates", + "letter": "t", + "count": 3 + }, + { + "word": "nonimmunity", + "letter": "t", + "count": 1 + }, + { + "word": "shifts", + "letter": "s", + "count": 2 + }, + { + "word": "one-eyed", + "letter": "e", + "count": 3 + }, + { + "word": "Tyrrheus", + "letter": "e", + "count": 1 + }, + { + "word": "wonder-seeking", + "letter": "n", + "count": 2 + }, + { + "word": "preequipment", + "letter": "e", + "count": 3 + }, + { + "word": "cotangential", + "letter": "o", + "count": 1 + }, + { + "word": "interaural", + "letter": "r", + "count": 2 + }, + { + "word": "nontumorous", + "letter": "o", + "count": 3 + }, + { + "word": "seigneuress", + "letter": "n", + "count": 1 + }, + { + "word": "abstemiousness", + "letter": "e", + "count": 2 + }, + { + "word": "bejeweling", + "letter": "e", + "count": 3 + }, + { + "word": "verticomental", + "letter": "o", + "count": 1 + }, + { + "word": "spokeless", + "letter": "e", + "count": 2 + }, + { + "word": "teasement", + "letter": "e", + "count": 3 + }, + { + "word": "Maletta", + "letter": "M", + "count": 1 + }, + { + "word": "oversolemnness", + "letter": "n", + "count": 2 + }, + { + "word": "Ustilaginaceae", + "letter": "a", + "count": 3 + }, + { + "word": "tracery", + "letter": "e", + "count": 1 + }, + { + "word": "Leviticus", + "letter": "i", + "count": 2 + }, + { + "word": "decalomania", + "letter": "a", + "count": 3 + }, + { + "word": "monades", + "letter": "a", + "count": 1 + }, + { + "word": "vortexes", + "letter": "e", + "count": 2 + }, + { + "word": "antiprostatic", + "letter": "t", + "count": 3 + }, + { + "word": "sonderclass", + "letter": "e", + "count": 1 + }, + { + "word": "composes", + "letter": "s", + "count": 2 + }, + { + "word": "cloud-drowned", + "letter": "d", + "count": 3 + }, + { + "word": "demove", + "letter": "o", + "count": 1 + }, + { + "word": "platitudinizer", + "letter": "t", + "count": 2 + }, + { + "word": "Labanna", + "letter": "a", + "count": 3 + }, + { + "word": "lobsters-claw", + "letter": "a", + "count": 1 + }, + { + "word": "bugout", + "letter": "u", + "count": 2 + }, + { + "word": "quarter-run", + "letter": "r", + "count": 3 + }, + { + "word": "subfrontal", + "letter": "a", + "count": 1 + }, + { + "word": "dog's-tooth", + "letter": "t", + "count": 2 + }, + { + "word": "blepharolithiasis", + "letter": "i", + "count": 3 + }, + { + "word": "fixatives", + "letter": "e", + "count": 1 + }, + { + "word": "finick", + "letter": "i", + "count": 2 + }, + { + "word": "superindignantly", + "letter": "n", + "count": 3 + }, + { + "word": "nonbigoted", + "letter": "g", + "count": 1 + }, + { + "word": "victoress", + "letter": "s", + "count": 2 + }, + { + "word": "cardiagra", + "letter": "a", + "count": 3 + }, + { + "word": "fucuses", + "letter": "c", + "count": 1 + }, + { + "word": "unuxorial", + "letter": "u", + "count": 2 + }, + { + "word": "undoneness", + "letter": "n", + "count": 3 + }, + { + "word": "unharmonically", + "letter": "y", + "count": 1 + }, + { + "word": "subcutaneously", + "letter": "s", + "count": 2 + }, + { + "word": "uncomplimenting", + "letter": "n", + "count": 3 + }, + { + "word": "ill-fitted", + "letter": "e", + "count": 1 + }, + { + "word": "piecework", + "letter": "e", + "count": 2 + }, + { + "word": "pretreatment", + "letter": "t", + "count": 3 + }, + { + "word": "untemporizing", + "letter": "e", + "count": 1 + }, + { + "word": "quick-falling", + "letter": "l", + "count": 2 + }, + { + "word": "unpervasiveness", + "letter": "s", + "count": 3 + }, + { + "word": "irradiancy", + "letter": "c", + "count": 1 + }, + { + "word": "self-limitation", + "letter": "l", + "count": 2 + }, + { + "word": "finickier", + "letter": "i", + "count": 3 + }, + { + "word": "unbush", + "letter": "b", + "count": 1 + }, + { + "word": "kerosine", + "letter": "e", + "count": 2 + }, + { + "word": "blackfellow", + "letter": "l", + "count": 3 + }, + { + "word": "syenogabbro", + "letter": "s", + "count": 1 + }, + { + "word": "swanwort", + "letter": "w", + "count": 2 + }, + { + "word": "unbibulously", + "letter": "u", + "count": 3 + }, + { + "word": "eradicates", + "letter": "i", + "count": 1 + }, + { + "word": "endogamies", + "letter": "e", + "count": 2 + }, + { + "word": "coregonoid", + "letter": "o", + "count": 3 + }, + { + "word": "brechtian", + "letter": "i", + "count": 1 + }, + { + "word": "rust-worn", + "letter": "r", + "count": 2 + }, + { + "word": "funiculus", + "letter": "u", + "count": 3 + }, + { + "word": "beseechingness", + "letter": "b", + "count": 1 + }, + { + "word": "half-quarterpace", + "letter": "e", + "count": 2 + }, + { + "word": "unmammonized", + "letter": "m", + "count": 3 + }, + { + "word": "tarsias", + "letter": "r", + "count": 1 + }, + { + "word": "pterygote", + "letter": "t", + "count": 2 + }, + { + "word": "stepbrotherhood", + "letter": "o", + "count": 3 + }, + { + "word": "antismut", + "letter": "i", + "count": 1 + }, + { + "word": "autolytic", + "letter": "t", + "count": 2 + }, + { + "word": "commotions", + "letter": "o", + "count": 3 + }, + { + "word": "Un-athenian", + "letter": "t", + "count": 1 + }, + { + "word": "superprecariousness", + "letter": "p", + "count": 2 + }, + { + "word": "electrotelethermometer", + "letter": "r", + "count": 3 + }, + { + "word": "winery", + "letter": "w", + "count": 1 + }, + { + "word": "lycopods", + "letter": "o", + "count": 2 + }, + { + "word": "zoopharmacological", + "letter": "a", + "count": 3 + }, + { + "word": "unconfounding", + "letter": "c", + "count": 1 + }, + { + "word": "polynomials", + "letter": "l", + "count": 2 + }, + { + "word": "reasonlessured", + "letter": "s", + "count": 3 + }, + { + "word": "Kohoutek", + "letter": "e", + "count": 1 + }, + { + "word": "fatuous", + "letter": "u", + "count": 2 + }, + { + "word": "transliterator", + "letter": "t", + "count": 3 + }, + { + "word": "fictionised", + "letter": "f", + "count": 1 + }, + { + "word": "nerveproof", + "letter": "e", + "count": 2 + }, + { + "word": "unconsolingly", + "letter": "n", + "count": 3 + }, + { + "word": "Oeonus", + "letter": "n", + "count": 1 + }, + { + "word": "nongrey", + "letter": "n", + "count": 2 + }, + { + "word": "captain-lieutenant", + "letter": "t", + "count": 3 + }, + { + "word": "philantomba", + "letter": "p", + "count": 1 + }, + { + "word": "tigress", + "letter": "s", + "count": 2 + }, + { + "word": "incandent", + "letter": "n", + "count": 3 + }, + { + "word": "Echinorhinus", + "letter": "r", + "count": 1 + }, + { + "word": "recompilement", + "letter": "m", + "count": 2 + }, + { + "word": "distomatosis", + "letter": "s", + "count": 3 + }, + { + "word": "aflatoxin", + "letter": "n", + "count": 1 + }, + { + "word": "strickling", + "letter": "i", + "count": 2 + }, + { + "word": "exaggerate", + "letter": "e", + "count": 3 + }, + { + "word": "Wangoni", + "letter": "g", + "count": 1 + }, + { + "word": "Brill", + "letter": "l", + "count": 2 + }, + { + "word": "toxicologist", + "letter": "o", + "count": 3 + }, + { + "word": "wedge-form", + "letter": "w", + "count": 1 + }, + { + "word": "entocondyle", + "letter": "e", + "count": 2 + }, + { + "word": "calabrasella", + "letter": "l", + "count": 3 + }, + { + "word": "Erickson", + "letter": "s", + "count": 1 + }, + { + "word": "Microspermae", + "letter": "e", + "count": 2 + }, + { + "word": "merry-totter", + "letter": "r", + "count": 3 + }, + { + "word": "hypopotassemic", + "letter": "m", + "count": 1 + }, + { + "word": "duffies", + "letter": "f", + "count": 2 + }, + { + "word": "neglectfulness", + "letter": "e", + "count": 3 + }, + { + "word": "Sessler", + "letter": "l", + "count": 1 + }, + { + "word": "homeotic", + "letter": "o", + "count": 2 + }, + { + "word": "preterrestrial", + "letter": "e", + "count": 3 + }, + { + "word": "denuded", + "letter": "n", + "count": 1 + }, + { + "word": "glandularly", + "letter": "a", + "count": 2 + }, + { + "word": "Gallaway", + "letter": "a", + "count": 3 + }, + { + "word": "liny", + "letter": "l", + "count": 1 + }, + { + "word": "cooner", + "letter": "o", + "count": 2 + }, + { + "word": "pantographically", + "letter": "a", + "count": 3 + }, + { + "word": "counter", + "letter": "o", + "count": 1 + }, + { + "word": "toothy-peg", + "letter": "t", + "count": 2 + }, + { + "word": "afterfermentation", + "letter": "t", + "count": 3 + }, + { + "word": "five-parted", + "letter": "v", + "count": 1 + }, + { + "word": "Echiurus", + "letter": "u", + "count": 2 + }, + { + "word": "Cananean", + "letter": "a", + "count": 3 + }, + { + "word": "Tarlton", + "letter": "t", + "count": 1 + }, + { + "word": "impropriate", + "letter": "i", + "count": 2 + }, + { + "word": "all-expense", + "letter": "e", + "count": 3 + }, + { + "word": "Obidicut", + "letter": "O", + "count": 1 + }, + { + "word": "synods", + "letter": "s", + "count": 2 + }, + { + "word": "superreward", + "letter": "r", + "count": 3 + }, + { + "word": "EMP", + "letter": "M", + "count": 1 + }, + { + "word": "relativistic", + "letter": "t", + "count": 2 + }, + { + "word": "Schaerbeek", + "letter": "e", + "count": 3 + }, + { + "word": "Tolstoyan", + "letter": "T", + "count": 1 + }, + { + "word": "chopper's", + "letter": "p", + "count": 2 + }, + { + "word": "centennially", + "letter": "n", + "count": 3 + }, + { + "word": "nonreliably", + "letter": "i", + "count": 1 + }, + { + "word": "black-whiskered", + "letter": "e", + "count": 2 + }, + { + "word": "multiple-series", + "letter": "e", + "count": 3 + }, + { + "word": "carvene", + "letter": "v", + "count": 1 + }, + { + "word": "Creole", + "letter": "e", + "count": 2 + }, + { + "word": "nonillion", + "letter": "n", + "count": 3 + }, + { + "word": "unprest", + "letter": "r", + "count": 1 + } +] \ No newline at end of file diff --git a/pipelinerl/counting/train_counting_problems.json b/pipelinerl/counting/train_counting_problems.json new file mode 100644 index 0000000..2b5cbe5 --- /dev/null +++ b/pipelinerl/counting/train_counting_problems.json @@ -0,0 +1,5002 @@ +[ + { + "word": "Cathlene", + "letter": "n", + "count": 1 + }, + { + "word": "twazzy", + "letter": "z", + "count": 2 + }, + { + "word": "uninterlinked", + "letter": "n", + "count": 3 + }, + { + "word": "housefurnishings", + "letter": "g", + "count": 1 + }, + { + "word": "dim-sensed", + "letter": "d", + "count": 2 + }, + { + "word": "people-pestered", + "letter": "p", + "count": 3 + }, + { + "word": "KCB", + "letter": "C", + "count": 1 + }, + { + "word": "upswelled", + "letter": "e", + "count": 2 + }, + { + "word": "odontotherapia", + "letter": "o", + "count": 3 + }, + { + "word": "assemble", + "letter": "l", + "count": 1 + }, + { + "word": "stintingly", + "letter": "i", + "count": 2 + }, + { + "word": "gassendist", + "letter": "s", + "count": 3 + }, + { + "word": "premeasurement", + "letter": "s", + "count": 1 + }, + { + "word": "yearend", + "letter": "e", + "count": 2 + }, + { + "word": "apprehensively", + "letter": "e", + "count": 3 + }, + { + "word": "preascetic", + "letter": "a", + "count": 1 + }, + { + "word": "unweighing", + "letter": "n", + "count": 2 + }, + { + "word": "well-exemplified", + "letter": "l", + "count": 3 + }, + { + "word": "Gavia", + "letter": "i", + "count": 1 + }, + { + "word": "analogism", + "letter": "a", + "count": 2 + }, + { + "word": "lease-pardle", + "letter": "e", + "count": 3 + }, + { + "word": "semiclinical", + "letter": "s", + "count": 1 + }, + { + "word": "splay-edged", + "letter": "d", + "count": 2 + }, + { + "word": "overgrainer", + "letter": "r", + "count": 3 + }, + { + "word": "wash-pot", + "letter": "p", + "count": 1 + }, + { + "word": "anglophiles", + "letter": "l", + "count": 2 + }, + { + "word": "scalpellum", + "letter": "l", + "count": 3 + }, + { + "word": "prelatic", + "letter": "t", + "count": 1 + }, + { + "word": "Wernsman", + "letter": "n", + "count": 2 + }, + { + "word": "dahabeahs", + "letter": "a", + "count": 3 + }, + { + "word": "idiotize", + "letter": "z", + "count": 1 + }, + { + "word": "unconvulsively", + "letter": "n", + "count": 2 + }, + { + "word": "tenementary", + "letter": "e", + "count": 3 + }, + { + "word": "reconstructionary", + "letter": "y", + "count": 1 + }, + { + "word": "bandman", + "letter": "a", + "count": 2 + }, + { + "word": "crassis", + "letter": "s", + "count": 3 + }, + { + "word": "Addressograph", + "letter": "o", + "count": 1 + }, + { + "word": "plaisanterie", + "letter": "i", + "count": 2 + }, + { + "word": "geelbeck", + "letter": "e", + "count": 3 + }, + { + "word": "antimetabole", + "letter": "l", + "count": 1 + }, + { + "word": "entogastric", + "letter": "t", + "count": 2 + }, + { + "word": "spectrobolograph", + "letter": "o", + "count": 3 + }, + { + "word": "subtropics", + "letter": "p", + "count": 1 + }, + { + "word": "bascunan", + "letter": "a", + "count": 2 + }, + { + "word": "cremaillere", + "letter": "e", + "count": 3 + }, + { + "word": "cathetus", + "letter": "u", + "count": 1 + }, + { + "word": "enamourment", + "letter": "m", + "count": 2 + }, + { + "word": "steeplejacks", + "letter": "e", + "count": 3 + }, + { + "word": "cuproammonium", + "letter": "n", + "count": 1 + }, + { + "word": "sure-set", + "letter": "e", + "count": 2 + }, + { + "word": "nonsexists", + "letter": "s", + "count": 3 + }, + { + "word": "endosmotic", + "letter": "n", + "count": 1 + }, + { + "word": "ebonites", + "letter": "e", + "count": 2 + }, + { + "word": "ever-changeful", + "letter": "e", + "count": 3 + }, + { + "word": "cecidiology", + "letter": "d", + "count": 1 + }, + { + "word": "four-poster", + "letter": "o", + "count": 2 + }, + { + "word": "quinanarii", + "letter": "i", + "count": 3 + }, + { + "word": "cachaemia", + "letter": "e", + "count": 1 + }, + { + "word": "unmaritime", + "letter": "m", + "count": 2 + }, + { + "word": "nonmorainic", + "letter": "n", + "count": 3 + }, + { + "word": "petrean", + "letter": "a", + "count": 1 + }, + { + "word": "Radnorshire", + "letter": "r", + "count": 2 + }, + { + "word": "Lasmarias", + "letter": "a", + "count": 3 + }, + { + "word": "cicisbeism", + "letter": "b", + "count": 1 + }, + { + "word": "Herophile", + "letter": "e", + "count": 2 + }, + { + "word": "scissor-tailed", + "letter": "s", + "count": 3 + }, + { + "word": "nonsensic", + "letter": "i", + "count": 1 + }, + { + "word": "carcinolysin", + "letter": "c", + "count": 2 + }, + { + "word": "possets", + "letter": "s", + "count": 3 + }, + { + "word": "senhoritas", + "letter": "o", + "count": 1 + }, + { + "word": "full-fledged", + "letter": "f", + "count": 2 + }, + { + "word": "green-barked", + "letter": "e", + "count": 3 + }, + { + "word": "right-angledness", + "letter": "r", + "count": 1 + }, + { + "word": "scentlessness", + "letter": "n", + "count": 2 + }, + { + "word": "sesames", + "letter": "s", + "count": 3 + }, + { + "word": "opisthoglyphous", + "letter": "i", + "count": 1 + }, + { + "word": "monoculist", + "letter": "o", + "count": 2 + }, + { + "word": "quarrelproof", + "letter": "r", + "count": 3 + }, + { + "word": "danes", + "letter": "d", + "count": 1 + }, + { + "word": "Jenness", + "letter": "e", + "count": 2 + }, + { + "word": "nonconsecutive", + "letter": "n", + "count": 3 + }, + { + "word": "eatable", + "letter": "b", + "count": 1 + }, + { + "word": "misrefer", + "letter": "r", + "count": 2 + }, + { + "word": "aerogenesis", + "letter": "e", + "count": 3 + }, + { + "word": "lazar-house", + "letter": "r", + "count": 1 + }, + { + "word": "piglinghood", + "letter": "g", + "count": 2 + }, + { + "word": "keennesses", + "letter": "s", + "count": 3 + }, + { + "word": "unfoolish", + "letter": "f", + "count": 1 + }, + { + "word": "puzzlers", + "letter": "z", + "count": 2 + }, + { + "word": "multiciliate", + "letter": "i", + "count": 3 + }, + { + "word": "phonophotoscopic", + "letter": "i", + "count": 1 + }, + { + "word": "reapplication", + "letter": "a", + "count": 2 + }, + { + "word": "misprinting", + "letter": "i", + "count": 3 + }, + { + "word": "scrubbiest", + "letter": "u", + "count": 1 + }, + { + "word": "niceling", + "letter": "n", + "count": 2 + }, + { + "word": "wave-encircled", + "letter": "e", + "count": 3 + }, + { + "word": "electrosurgeries", + "letter": "l", + "count": 1 + }, + { + "word": "eggeater", + "letter": "g", + "count": 2 + }, + { + "word": "callownesses", + "letter": "s", + "count": 3 + }, + { + "word": "nongaseousness", + "letter": "g", + "count": 1 + }, + { + "word": "nonspirited", + "letter": "n", + "count": 2 + }, + { + "word": "ferrophosphorus", + "letter": "r", + "count": 3 + }, + { + "word": "Ihlen", + "letter": "n", + "count": 1 + }, + { + "word": "Hydropterideae", + "letter": "r", + "count": 2 + }, + { + "word": "breeze's", + "letter": "e", + "count": 3 + }, + { + "word": "unsolicitude", + "letter": "s", + "count": 1 + }, + { + "word": "Hallie", + "letter": "l", + "count": 2 + }, + { + "word": "retrofire", + "letter": "r", + "count": 3 + }, + { + "word": "drugs", + "letter": "s", + "count": 1 + }, + { + "word": "megaphone", + "letter": "e", + "count": 2 + }, + { + "word": "fossilised", + "letter": "s", + "count": 3 + }, + { + "word": "ataxias", + "letter": "i", + "count": 1 + }, + { + "word": "unazotized", + "letter": "z", + "count": 2 + }, + { + "word": "noncounteractive", + "letter": "n", + "count": 3 + }, + { + "word": "Angell", + "letter": "g", + "count": 1 + }, + { + "word": "archhumbug", + "letter": "u", + "count": 2 + }, + { + "word": "Nadabas", + "letter": "a", + "count": 3 + }, + { + "word": "Restany", + "letter": "n", + "count": 1 + }, + { + "word": "eburnation", + "letter": "n", + "count": 2 + }, + { + "word": "overexuberant", + "letter": "e", + "count": 3 + }, + { + "word": "trisodium", + "letter": "o", + "count": 1 + }, + { + "word": "wiggeries", + "letter": "i", + "count": 2 + }, + { + "word": "narcosynthesis", + "letter": "s", + "count": 3 + }, + { + "word": "perivenous", + "letter": "u", + "count": 1 + }, + { + "word": "rhamnuses", + "letter": "s", + "count": 2 + }, + { + "word": "coxarthrocace", + "letter": "c", + "count": 3 + }, + { + "word": "Jacobinically", + "letter": "b", + "count": 1 + }, + { + "word": "back-pedaling", + "letter": "a", + "count": 2 + }, + { + "word": "solonetses", + "letter": "s", + "count": 3 + }, + { + "word": "staider", + "letter": "r", + "count": 1 + }, + { + "word": "bepowder", + "letter": "e", + "count": 2 + }, + { + "word": "glorification", + "letter": "i", + "count": 3 + }, + { + "word": "chronically", + "letter": "n", + "count": 1 + }, + { + "word": "anesthetist", + "letter": "s", + "count": 2 + }, + { + "word": "seven-ounce", + "letter": "e", + "count": 3 + }, + { + "word": "crosscurrented", + "letter": "t", + "count": 1 + }, + { + "word": "chumpaka", + "letter": "a", + "count": 2 + }, + { + "word": "eugenolate", + "letter": "e", + "count": 3 + }, + { + "word": "linkmen", + "letter": "i", + "count": 1 + }, + { + "word": "Ningirsu", + "letter": "i", + "count": 2 + }, + { + "word": "amadan", + "letter": "a", + "count": 3 + }, + { + "word": "pump-handler", + "letter": "l", + "count": 1 + }, + { + "word": "infragrant", + "letter": "n", + "count": 2 + }, + { + "word": "analysability", + "letter": "a", + "count": 3 + }, + { + "word": "cocorico", + "letter": "i", + "count": 1 + }, + { + "word": "Ectopistes", + "letter": "t", + "count": 2 + }, + { + "word": "edge-grained", + "letter": "e", + "count": 3 + }, + { + "word": "raging", + "letter": "r", + "count": 1 + }, + { + "word": "clarigold", + "letter": "l", + "count": 2 + }, + { + "word": "beefer", + "letter": "e", + "count": 3 + }, + { + "word": "remagnifying", + "letter": "m", + "count": 1 + }, + { + "word": "preterminal", + "letter": "e", + "count": 2 + }, + { + "word": "urutu", + "letter": "u", + "count": 3 + }, + { + "word": "okayed", + "letter": "d", + "count": 1 + }, + { + "word": "accommodator", + "letter": "c", + "count": 2 + }, + { + "word": "amidosuccinamic", + "letter": "c", + "count": 3 + }, + { + "word": "nonterminating", + "letter": "e", + "count": 1 + }, + { + "word": "flagfish", + "letter": "f", + "count": 2 + }, + { + "word": "pusillanimity", + "letter": "i", + "count": 3 + }, + { + "word": "shipfitter", + "letter": "e", + "count": 1 + }, + { + "word": "swept-forward", + "letter": "r", + "count": 2 + }, + { + "word": "unminimising", + "letter": "n", + "count": 3 + }, + { + "word": "antonym", + "letter": "t", + "count": 1 + }, + { + "word": "Dartagnan", + "letter": "n", + "count": 2 + }, + { + "word": "zeekoe", + "letter": "e", + "count": 3 + }, + { + "word": "transitionalness", + "letter": "l", + "count": 1 + }, + { + "word": "aikido", + "letter": "i", + "count": 2 + }, + { + "word": "lecherousness", + "letter": "e", + "count": 3 + }, + { + "word": "supersized", + "letter": "u", + "count": 1 + }, + { + "word": "autographic", + "letter": "a", + "count": 2 + }, + { + "word": "prohibition-proof", + "letter": "i", + "count": 3 + }, + { + "word": "sleepwort", + "letter": "t", + "count": 1 + }, + { + "word": "opifice", + "letter": "i", + "count": 2 + }, + { + "word": "nonengagement", + "letter": "e", + "count": 3 + }, + { + "word": "incalculableness", + "letter": "i", + "count": 1 + }, + { + "word": "quaking-grass", + "letter": "a", + "count": 2 + }, + { + "word": "myometrium", + "letter": "m", + "count": 3 + }, + { + "word": "Loffler", + "letter": "L", + "count": 1 + }, + { + "word": "Koniaga", + "letter": "a", + "count": 2 + }, + { + "word": "Keokee", + "letter": "e", + "count": 3 + }, + { + "word": "curtsy's", + "letter": "c", + "count": 1 + }, + { + "word": "entremets", + "letter": "t", + "count": 2 + }, + { + "word": "conditioning", + "letter": "n", + "count": 3 + }, + { + "word": "centrodesmose", + "letter": "r", + "count": 1 + }, + { + "word": "renames", + "letter": "e", + "count": 2 + }, + { + "word": "nonindurative", + "letter": "n", + "count": 3 + }, + { + "word": "intolerating", + "letter": "r", + "count": 1 + }, + { + "word": "flagrance", + "letter": "a", + "count": 2 + }, + { + "word": "mesmerizable", + "letter": "e", + "count": 3 + }, + { + "word": "inconsumable", + "letter": "e", + "count": 1 + }, + { + "word": "antioxidase", + "letter": "a", + "count": 2 + }, + { + "word": "commenceable", + "letter": "e", + "count": 3 + }, + { + "word": "outlighten", + "letter": "g", + "count": 1 + }, + { + "word": "denudations", + "letter": "d", + "count": 2 + }, + { + "word": "heptene", + "letter": "e", + "count": 3 + }, + { + "word": "epigrammatised", + "letter": "d", + "count": 1 + }, + { + "word": "Coccidae", + "letter": "c", + "count": 2 + }, + { + "word": "cricoidectomy", + "letter": "c", + "count": 3 + }, + { + "word": "forceps", + "letter": "o", + "count": 1 + }, + { + "word": "Neidhardt", + "letter": "d", + "count": 2 + }, + { + "word": "overclement", + "letter": "e", + "count": 3 + }, + { + "word": "zaffree", + "letter": "r", + "count": 1 + }, + { + "word": "encomiastical", + "letter": "a", + "count": 2 + }, + { + "word": "half-volley", + "letter": "l", + "count": 3 + }, + { + "word": "Trochodendraceae", + "letter": "T", + "count": 1 + }, + { + "word": "word-group", + "letter": "r", + "count": 2 + }, + { + "word": "contratulations", + "letter": "t", + "count": 3 + }, + { + "word": "eyereach", + "letter": "h", + "count": 1 + }, + { + "word": "uniformitarian", + "letter": "a", + "count": 2 + }, + { + "word": "progesterone", + "letter": "e", + "count": 3 + }, + { + "word": "polyaffectioned", + "letter": "i", + "count": 1 + }, + { + "word": "Fabiano", + "letter": "a", + "count": 2 + }, + { + "word": "ratatat", + "letter": "t", + "count": 3 + }, + { + "word": "provaccinist", + "letter": "v", + "count": 1 + }, + { + "word": "exteriors", + "letter": "r", + "count": 2 + }, + { + "word": "palabra", + "letter": "a", + "count": 3 + }, + { + "word": "Sevan", + "letter": "a", + "count": 1 + }, + { + "word": "nonart", + "letter": "n", + "count": 2 + }, + { + "word": "unenkindled", + "letter": "n", + "count": 3 + }, + { + "word": "moisteners", + "letter": "t", + "count": 1 + }, + { + "word": "unwilledness", + "letter": "l", + "count": 2 + }, + { + "word": "odd-come-shortly", + "letter": "o", + "count": 3 + }, + { + "word": "bronzitite", + "letter": "r", + "count": 1 + }, + { + "word": "bedouse", + "letter": "e", + "count": 2 + }, + { + "word": "hysteroptosis", + "letter": "s", + "count": 3 + }, + { + "word": "Maplecrest", + "letter": "a", + "count": 1 + }, + { + "word": "ornithocephalous", + "letter": "h", + "count": 2 + }, + { + "word": "thought-stirring", + "letter": "t", + "count": 3 + }, + { + "word": "calycanthin", + "letter": "l", + "count": 1 + }, + { + "word": "bearer", + "letter": "e", + "count": 2 + }, + { + "word": "uninitialed", + "letter": "i", + "count": 3 + }, + { + "word": "nonexaction", + "letter": "t", + "count": 1 + }, + { + "word": "subturbary", + "letter": "b", + "count": 2 + }, + { + "word": "wide-crested", + "letter": "e", + "count": 3 + }, + { + "word": "Raskin", + "letter": "n", + "count": 1 + }, + { + "word": "monorhythmic", + "letter": "o", + "count": 2 + }, + { + "word": "musicophilosophical", + "letter": "i", + "count": 3 + }, + { + "word": "green-sheathed", + "letter": "n", + "count": 1 + }, + { + "word": "gata", + "letter": "a", + "count": 2 + }, + { + "word": "philanthropically", + "letter": "l", + "count": 3 + }, + { + "word": "Urfa", + "letter": "f", + "count": 1 + }, + { + "word": "baronetcies", + "letter": "e", + "count": 2 + }, + { + "word": "inequipotentiality", + "letter": "t", + "count": 3 + }, + { + "word": "unfalsity", + "letter": "s", + "count": 1 + }, + { + "word": "Pini", + "letter": "i", + "count": 2 + }, + { + "word": "peregrinative", + "letter": "e", + "count": 3 + }, + { + "word": "overfrailty", + "letter": "i", + "count": 1 + }, + { + "word": "monolithic", + "letter": "o", + "count": 2 + }, + { + "word": "saxifragaceous", + "letter": "a", + "count": 3 + }, + { + "word": "snake-winged", + "letter": "k", + "count": 1 + }, + { + "word": "Gilbertsville", + "letter": "i", + "count": 2 + }, + { + "word": "feverfews", + "letter": "e", + "count": 3 + }, + { + "word": "muruxi", + "letter": "i", + "count": 1 + }, + { + "word": "gainfulness", + "letter": "s", + "count": 2 + }, + { + "word": "star-scattered", + "letter": "t", + "count": 3 + }, + { + "word": "unloading", + "letter": "o", + "count": 1 + }, + { + "word": "sulphoricinoleic", + "letter": "l", + "count": 2 + }, + { + "word": "reedmaker", + "letter": "e", + "count": 3 + }, + { + "word": "marlings", + "letter": "r", + "count": 1 + }, + { + "word": "fictionisation", + "letter": "o", + "count": 2 + }, + { + "word": "nonuniformities", + "letter": "n", + "count": 3 + }, + { + "word": "hematite", + "letter": "h", + "count": 1 + }, + { + "word": "Zinfandel", + "letter": "n", + "count": 2 + }, + { + "word": "chair-warmer", + "letter": "r", + "count": 3 + }, + { + "word": "suspicion", + "letter": "u", + "count": 1 + }, + { + "word": "Callianassa", + "letter": "l", + "count": 2 + }, + { + "word": "mononomian", + "letter": "n", + "count": 3 + }, + { + "word": "Chloromycetin", + "letter": "n", + "count": 1 + }, + { + "word": "plumbership", + "letter": "p", + "count": 2 + }, + { + "word": "amyelencephalous", + "letter": "e", + "count": 3 + }, + { + "word": "indagate", + "letter": "t", + "count": 1 + }, + { + "word": "unreconstructible", + "letter": "r", + "count": 2 + }, + { + "word": "limitaries", + "letter": "i", + "count": 3 + }, + { + "word": "calorification", + "letter": "t", + "count": 1 + }, + { + "word": "bronchadenitis", + "letter": "i", + "count": 2 + }, + { + "word": "oversee", + "letter": "e", + "count": 3 + }, + { + "word": "golfed", + "letter": "f", + "count": 1 + }, + { + "word": "layoff", + "letter": "f", + "count": 2 + }, + { + "word": "flaggella", + "letter": "l", + "count": 3 + }, + { + "word": "Jacobs", + "letter": "J", + "count": 1 + }, + { + "word": "amphithurthura", + "letter": "t", + "count": 2 + }, + { + "word": "consonantally", + "letter": "n", + "count": 3 + }, + { + "word": "plitch", + "letter": "p", + "count": 1 + }, + { + "word": "mizzenmast", + "letter": "z", + "count": 2 + }, + { + "word": "congealableness", + "letter": "e", + "count": 3 + }, + { + "word": "photozincograph", + "letter": "t", + "count": 1 + }, + { + "word": "mesprise", + "letter": "s", + "count": 2 + }, + { + "word": "nonperversities", + "letter": "e", + "count": 3 + }, + { + "word": "recomparison", + "letter": "n", + "count": 1 + }, + { + "word": "polyose", + "letter": "o", + "count": 2 + }, + { + "word": "acetmethylanilide", + "letter": "e", + "count": 3 + }, + { + "word": "kludge", + "letter": "k", + "count": 1 + }, + { + "word": "nonrevealing", + "letter": "e", + "count": 2 + }, + { + "word": "frequentest", + "letter": "e", + "count": 3 + }, + { + "word": "beneurous", + "letter": "r", + "count": 1 + }, + { + "word": "raptureless", + "letter": "r", + "count": 2 + }, + { + "word": "stalactite", + "letter": "t", + "count": 3 + }, + { + "word": "oofier", + "letter": "f", + "count": 1 + }, + { + "word": "vagabondager", + "letter": "g", + "count": 2 + }, + { + "word": "protodont", + "letter": "o", + "count": 3 + }, + { + "word": "edulcorate", + "letter": "d", + "count": 1 + }, + { + "word": "desolated", + "letter": "e", + "count": 2 + }, + { + "word": "Kauravas", + "letter": "a", + "count": 3 + }, + { + "word": "trencherwise", + "letter": "t", + "count": 1 + }, + { + "word": "ugly-looking", + "letter": "l", + "count": 2 + }, + { + "word": "reunification", + "letter": "i", + "count": 3 + }, + { + "word": "purpled", + "letter": "e", + "count": 1 + }, + { + "word": "ornithophilous", + "letter": "h", + "count": 2 + }, + { + "word": "Saint-estephe", + "letter": "e", + "count": 3 + }, + { + "word": "powderies", + "letter": "d", + "count": 1 + }, + { + "word": "two-cent", + "letter": "t", + "count": 2 + }, + { + "word": "acappella", + "letter": "a", + "count": 3 + }, + { + "word": "suggestively", + "letter": "u", + "count": 1 + }, + { + "word": "Follett", + "letter": "t", + "count": 2 + }, + { + "word": "rachianesthesia", + "letter": "a", + "count": 3 + }, + { + "word": "malcontents", + "letter": "m", + "count": 1 + }, + { + "word": "Kolosick", + "letter": "o", + "count": 2 + }, + { + "word": "superextremeness", + "letter": "s", + "count": 3 + }, + { + "word": "Gackle", + "letter": "e", + "count": 1 + }, + { + "word": "perspirability", + "letter": "p", + "count": 2 + }, + { + "word": "orthophenylene", + "letter": "e", + "count": 3 + }, + { + "word": "certified", + "letter": "r", + "count": 1 + }, + { + "word": "episepalous", + "letter": "p", + "count": 2 + }, + { + "word": "sossiego", + "letter": "s", + "count": 3 + }, + { + "word": "lv.", + "letter": "v", + "count": 1 + }, + { + "word": "Crocodylidae", + "letter": "d", + "count": 2 + }, + { + "word": "cockatoo", + "letter": "o", + "count": 3 + }, + { + "word": "colubroid", + "letter": "i", + "count": 1 + }, + { + "word": "Manabozho", + "letter": "a", + "count": 2 + }, + { + "word": "theologization", + "letter": "o", + "count": 3 + }, + { + "word": "precipitantly", + "letter": "e", + "count": 1 + }, + { + "word": "flubber", + "letter": "b", + "count": 2 + }, + { + "word": "teethache", + "letter": "e", + "count": 3 + }, + { + "word": "De-hellenize", + "letter": "-", + "count": 1 + }, + { + "word": "longboat", + "letter": "o", + "count": 2 + }, + { + "word": "monosubstituted", + "letter": "t", + "count": 3 + }, + { + "word": "overstayal", + "letter": "s", + "count": 1 + }, + { + "word": "endotheliolytic", + "letter": "o", + "count": 2 + }, + { + "word": "rangatira", + "letter": "a", + "count": 3 + }, + { + "word": "pretested", + "letter": "r", + "count": 1 + }, + { + "word": "axoplasmic", + "letter": "a", + "count": 2 + }, + { + "word": "non-combatant", + "letter": "n", + "count": 3 + }, + { + "word": "caressable", + "letter": "l", + "count": 1 + }, + { + "word": "granat", + "letter": "a", + "count": 2 + }, + { + "word": "newscasters", + "letter": "s", + "count": 3 + }, + { + "word": "unprejudiciable", + "letter": "l", + "count": 1 + }, + { + "word": "milliner", + "letter": "i", + "count": 2 + }, + { + "word": "incommodation", + "letter": "o", + "count": 3 + }, + { + "word": "yellow-dyed", + "letter": "-", + "count": 1 + }, + { + "word": "untouchably", + "letter": "u", + "count": 2 + }, + { + "word": "camleteen", + "letter": "e", + "count": 3 + }, + { + "word": "stonemasonry", + "letter": "r", + "count": 1 + }, + { + "word": "unloanably", + "letter": "l", + "count": 2 + }, + { + "word": "lapilliform", + "letter": "l", + "count": 3 + }, + { + "word": "inquilinism", + "letter": "m", + "count": 1 + }, + { + "word": "brickmaker", + "letter": "r", + "count": 2 + }, + { + "word": "Donoho", + "letter": "o", + "count": 3 + }, + { + "word": "sailye", + "letter": "s", + "count": 1 + }, + { + "word": "Woodring", + "letter": "o", + "count": 2 + }, + { + "word": "concubinarian", + "letter": "n", + "count": 3 + }, + { + "word": "Wildomar", + "letter": "a", + "count": 1 + }, + { + "word": "transistorizes", + "letter": "r", + "count": 2 + }, + { + "word": "subenfeoff", + "letter": "f", + "count": 3 + }, + { + "word": "Hathorne", + "letter": "o", + "count": 1 + }, + { + "word": "veniremen", + "letter": "n", + "count": 2 + }, + { + "word": "scintillize", + "letter": "i", + "count": 3 + }, + { + "word": "Dyophysite", + "letter": "D", + "count": 1 + }, + { + "word": "fitty", + "letter": "t", + "count": 2 + }, + { + "word": "egocentricities", + "letter": "i", + "count": 3 + }, + { + "word": "larmier", + "letter": "a", + "count": 1 + }, + { + "word": "subsecute", + "letter": "s", + "count": 2 + }, + { + "word": "calcitic", + "letter": "c", + "count": 3 + }, + { + "word": "Withers", + "letter": "i", + "count": 1 + }, + { + "word": "squarishly", + "letter": "s", + "count": 2 + }, + { + "word": "biostatistic", + "letter": "i", + "count": 3 + }, + { + "word": "rediscounted", + "letter": "i", + "count": 1 + }, + { + "word": "underzeal", + "letter": "e", + "count": 2 + }, + { + "word": "deoxidized", + "letter": "d", + "count": 3 + }, + { + "word": "Malebolge", + "letter": "a", + "count": 1 + }, + { + "word": "psaltes", + "letter": "s", + "count": 2 + }, + { + "word": "zootomies", + "letter": "o", + "count": 3 + }, + { + "word": "lyophiled", + "letter": "p", + "count": 1 + }, + { + "word": "ponderative", + "letter": "e", + "count": 2 + }, + { + "word": "nondiffidently", + "letter": "n", + "count": 3 + }, + { + "word": "nota", + "letter": "n", + "count": 1 + }, + { + "word": "concertment", + "letter": "n", + "count": 2 + }, + { + "word": "fine-leaved", + "letter": "e", + "count": 3 + }, + { + "word": "telangiectatic", + "letter": "g", + "count": 1 + }, + { + "word": "two-minded", + "letter": "d", + "count": 2 + }, + { + "word": "saturnalianly", + "letter": "a", + "count": 3 + }, + { + "word": "Idonah", + "letter": "d", + "count": 1 + }, + { + "word": "unaffecting", + "letter": "n", + "count": 2 + }, + { + "word": "propodeon", + "letter": "o", + "count": 3 + }, + { + "word": "agoho", + "letter": "a", + "count": 1 + }, + { + "word": "Pro-bonapartean", + "letter": "n", + "count": 2 + }, + { + "word": "stereotype", + "letter": "e", + "count": 3 + }, + { + "word": "Nobel", + "letter": "b", + "count": 1 + }, + { + "word": "backstabber", + "letter": "a", + "count": 2 + }, + { + "word": "plasmatoparous", + "letter": "a", + "count": 3 + }, + { + "word": "confiscation", + "letter": "f", + "count": 1 + }, + { + "word": "ovaritis", + "letter": "i", + "count": 2 + }, + { + "word": "dissimilated", + "letter": "i", + "count": 3 + }, + { + "word": "enfoldment", + "letter": "t", + "count": 1 + }, + { + "word": "humerocubital", + "letter": "u", + "count": 2 + }, + { + "word": "establishment's", + "letter": "s", + "count": 3 + }, + { + "word": "thrombins", + "letter": "h", + "count": 1 + }, + { + "word": "grimier", + "letter": "r", + "count": 2 + }, + { + "word": "Tammanial", + "letter": "a", + "count": 3 + }, + { + "word": "phantom's", + "letter": "t", + "count": 1 + }, + { + "word": "Tasia", + "letter": "a", + "count": 2 + }, + { + "word": "lowbell", + "letter": "l", + "count": 3 + }, + { + "word": "ungrappled", + "letter": "l", + "count": 1 + }, + { + "word": "forleave", + "letter": "e", + "count": 2 + }, + { + "word": "unfringing", + "letter": "n", + "count": 3 + }, + { + "word": "bewhistle", + "letter": "l", + "count": 1 + }, + { + "word": "doblones", + "letter": "o", + "count": 2 + }, + { + "word": "infeoffs", + "letter": "f", + "count": 3 + }, + { + "word": "quickness", + "letter": "i", + "count": 1 + }, + { + "word": "earth-light", + "letter": "h", + "count": 2 + }, + { + "word": "taeninidia", + "letter": "i", + "count": 3 + }, + { + "word": "Bhil", + "letter": "i", + "count": 1 + }, + { + "word": "strident-voiced", + "letter": "e", + "count": 2 + }, + { + "word": "untempered", + "letter": "e", + "count": 3 + }, + { + "word": "reaffirmations", + "letter": "o", + "count": 1 + }, + { + "word": "tapeman", + "letter": "a", + "count": 2 + }, + { + "word": "unavenging", + "letter": "n", + "count": 3 + }, + { + "word": "unfeminize", + "letter": "z", + "count": 1 + }, + { + "word": "dentist's", + "letter": "s", + "count": 2 + }, + { + "word": "nontransformation", + "letter": "o", + "count": 3 + }, + { + "word": "Rifton", + "letter": "o", + "count": 1 + }, + { + "word": "gyneolatry", + "letter": "y", + "count": 2 + }, + { + "word": "merosomatous", + "letter": "o", + "count": 3 + }, + { + "word": "unproducibleness", + "letter": "i", + "count": 1 + }, + { + "word": "phenelzine", + "letter": "n", + "count": 2 + }, + { + "word": "monostrophics", + "letter": "o", + "count": 3 + }, + { + "word": "yirk", + "letter": "i", + "count": 1 + }, + { + "word": "clipping's", + "letter": "i", + "count": 2 + }, + { + "word": "stintless", + "letter": "s", + "count": 3 + }, + { + "word": "xenoparasite", + "letter": "n", + "count": 1 + }, + { + "word": "acinetic", + "letter": "c", + "count": 2 + }, + { + "word": "re-enlighten", + "letter": "e", + "count": 3 + }, + { + "word": "procommunism", + "letter": "p", + "count": 1 + }, + { + "word": "epidictical", + "letter": "c", + "count": 2 + }, + { + "word": "antivandalism", + "letter": "a", + "count": 3 + }, + { + "word": "sesame", + "letter": "a", + "count": 1 + }, + { + "word": "assertingly", + "letter": "s", + "count": 2 + }, + { + "word": "maxillojugal", + "letter": "l", + "count": 3 + }, + { + "word": "infusing", + "letter": "g", + "count": 1 + }, + { + "word": "rejumble", + "letter": "e", + "count": 2 + }, + { + "word": "miracidial", + "letter": "i", + "count": 3 + }, + { + "word": "easement's", + "letter": "'", + "count": 1 + }, + { + "word": "dramatizes", + "letter": "a", + "count": 2 + }, + { + "word": "paleoentomological", + "letter": "l", + "count": 3 + }, + { + "word": "helpmates", + "letter": "l", + "count": 1 + }, + { + "word": "Germann", + "letter": "n", + "count": 2 + }, + { + "word": "plasmaphaeresis", + "letter": "a", + "count": 3 + }, + { + "word": "larceners", + "letter": "l", + "count": 1 + }, + { + "word": "exclosure", + "letter": "e", + "count": 2 + }, + { + "word": "never-ready", + "letter": "e", + "count": 3 + }, + { + "word": "floody", + "letter": "d", + "count": 1 + }, + { + "word": "beveil", + "letter": "e", + "count": 2 + }, + { + "word": "tampala", + "letter": "a", + "count": 3 + }, + { + "word": "unmarriageable", + "letter": "b", + "count": 1 + }, + { + "word": "diffractive", + "letter": "i", + "count": 2 + }, + { + "word": "microprojection", + "letter": "o", + "count": 3 + }, + { + "word": "Eastre", + "letter": "t", + "count": 1 + }, + { + "word": "Yankee", + "letter": "e", + "count": 2 + }, + { + "word": "solenesses", + "letter": "e", + "count": 3 + }, + { + "word": "aorta", + "letter": "r", + "count": 1 + }, + { + "word": "supersuperabundance", + "letter": "r", + "count": 2 + }, + { + "word": "world-traveler", + "letter": "r", + "count": 3 + }, + { + "word": "nonproductive", + "letter": "i", + "count": 1 + }, + { + "word": "vivisected", + "letter": "i", + "count": 2 + }, + { + "word": "assets", + "letter": "s", + "count": 3 + }, + { + "word": "sufficingly", + "letter": "u", + "count": 1 + }, + { + "word": "vacuum-clean", + "letter": "c", + "count": 2 + }, + { + "word": "Bothriocidaris", + "letter": "i", + "count": 3 + }, + { + "word": "biune", + "letter": "n", + "count": 1 + }, + { + "word": "acropetal", + "letter": "a", + "count": 2 + }, + { + "word": "duodenoenterostomy", + "letter": "e", + "count": 3 + }, + { + "word": "gobstick", + "letter": "i", + "count": 1 + }, + { + "word": "counter-earth", + "letter": "e", + "count": 2 + }, + { + "word": "serologically", + "letter": "l", + "count": 3 + }, + { + "word": "preconceals", + "letter": "l", + "count": 1 + }, + { + "word": "self-perfecting", + "letter": "f", + "count": 2 + }, + { + "word": "nonhistrionical", + "letter": "i", + "count": 3 + }, + { + "word": "bariolage", + "letter": "r", + "count": 1 + }, + { + "word": "nonion", + "letter": "o", + "count": 2 + }, + { + "word": "aerogenesis", + "letter": "e", + "count": 3 + }, + { + "word": "kipped", + "letter": "e", + "count": 1 + }, + { + "word": "emptysis", + "letter": "s", + "count": 2 + }, + { + "word": "dyeweeds", + "letter": "e", + "count": 3 + }, + { + "word": "iour", + "letter": "o", + "count": 1 + }, + { + "word": "narcomaniac", + "letter": "c", + "count": 2 + }, + { + "word": "cockle-headed", + "letter": "e", + "count": 3 + }, + { + "word": "guilt-feelings", + "letter": "-", + "count": 1 + }, + { + "word": "collar-cutting", + "letter": "c", + "count": 2 + }, + { + "word": "toxicologic", + "letter": "o", + "count": 3 + }, + { + "word": "Gottlander", + "letter": "e", + "count": 1 + }, + { + "word": "Rigsdag", + "letter": "g", + "count": 2 + }, + { + "word": "Scatophagidae", + "letter": "a", + "count": 3 + }, + { + "word": "ungarrulousness", + "letter": "e", + "count": 1 + }, + { + "word": "lumisterol", + "letter": "l", + "count": 2 + }, + { + "word": "unulcerously", + "letter": "u", + "count": 3 + }, + { + "word": "sore-footed", + "letter": "r", + "count": 1 + }, + { + "word": "tumulary", + "letter": "u", + "count": 2 + }, + { + "word": "inimically", + "letter": "i", + "count": 3 + }, + { + "word": "Lawrence", + "letter": "c", + "count": 1 + }, + { + "word": "creature's", + "letter": "r", + "count": 2 + }, + { + "word": "oversuspiciously", + "letter": "s", + "count": 3 + }, + { + "word": "soporiferously", + "letter": "y", + "count": 1 + }, + { + "word": "octocentennial", + "letter": "c", + "count": 2 + }, + { + "word": "reinfiltrating", + "letter": "i", + "count": 3 + }, + { + "word": "coworking", + "letter": "k", + "count": 1 + }, + { + "word": "episcopised", + "letter": "p", + "count": 2 + }, + { + "word": "nephrectomies", + "letter": "e", + "count": 3 + }, + { + "word": "Cryptostomata", + "letter": "p", + "count": 1 + }, + { + "word": "examination", + "letter": "n", + "count": 2 + }, + { + "word": "hemiscotosis", + "letter": "s", + "count": 3 + }, + { + "word": "integrand", + "letter": "t", + "count": 1 + }, + { + "word": "postboy", + "letter": "o", + "count": 2 + }, + { + "word": "unpresented", + "letter": "e", + "count": 3 + }, + { + "word": "teknonymous", + "letter": "e", + "count": 1 + }, + { + "word": "exocannibalism", + "letter": "i", + "count": 2 + }, + { + "word": "zoophobes", + "letter": "o", + "count": 3 + }, + { + "word": "chondria", + "letter": "i", + "count": 1 + }, + { + "word": "Hyeres", + "letter": "e", + "count": 2 + }, + { + "word": "nonnautically", + "letter": "n", + "count": 3 + }, + { + "word": "pneumococcic", + "letter": "i", + "count": 1 + }, + { + "word": "lovelies", + "letter": "e", + "count": 2 + }, + { + "word": "unmelodically", + "letter": "l", + "count": 3 + }, + { + "word": "cavalero", + "letter": "e", + "count": 1 + }, + { + "word": "uninvidious", + "letter": "u", + "count": 2 + }, + { + "word": "Pseudo-isidorian", + "letter": "i", + "count": 3 + }, + { + "word": "Peterkin", + "letter": "P", + "count": 1 + }, + { + "word": "bimillennia", + "letter": "l", + "count": 2 + }, + { + "word": "parapathy", + "letter": "a", + "count": 3 + }, + { + "word": "collapsing", + "letter": "n", + "count": 1 + }, + { + "word": "stock-still", + "letter": "s", + "count": 2 + }, + { + "word": "noisomeness", + "letter": "s", + "count": 3 + }, + { + "word": "recentness", + "letter": "c", + "count": 1 + }, + { + "word": "koppa", + "letter": "p", + "count": 2 + }, + { + "word": "oxbloods", + "letter": "o", + "count": 3 + }, + { + "word": "refabricate", + "letter": "f", + "count": 1 + }, + { + "word": "bristling", + "letter": "i", + "count": 2 + }, + { + "word": "self-controller", + "letter": "l", + "count": 3 + }, + { + "word": "mooress", + "letter": "r", + "count": 1 + }, + { + "word": "angiokinesis", + "letter": "n", + "count": 2 + }, + { + "word": "interpledged", + "letter": "e", + "count": 3 + }, + { + "word": "carpetwoven", + "letter": "t", + "count": 1 + }, + { + "word": "Quinquagesima", + "letter": "u", + "count": 2 + }, + { + "word": "conferrer", + "letter": "r", + "count": 3 + }, + { + "word": "panatelas", + "letter": "e", + "count": 1 + }, + { + "word": "hernioid", + "letter": "i", + "count": 2 + }, + { + "word": "uncommemorative", + "letter": "m", + "count": 3 + }, + { + "word": "pearlsides", + "letter": "l", + "count": 1 + }, + { + "word": "hole-and-corner", + "letter": "r", + "count": 2 + }, + { + "word": "syllabicness", + "letter": "s", + "count": 3 + }, + { + "word": "stoke", + "letter": "o", + "count": 1 + }, + { + "word": "lack-lustre", + "letter": "l", + "count": 2 + }, + { + "word": "traditionaries", + "letter": "i", + "count": 3 + }, + { + "word": "hangie", + "letter": "a", + "count": 1 + }, + { + "word": "elderly", + "letter": "l", + "count": 2 + }, + { + "word": "octochord", + "letter": "o", + "count": 3 + }, + { + "word": "searlesite", + "letter": "t", + "count": 1 + }, + { + "word": "altiplanicie", + "letter": "l", + "count": 2 + }, + { + "word": "insatisfactorily", + "letter": "i", + "count": 3 + }, + { + "word": "well-browned", + "letter": "r", + "count": 1 + }, + { + "word": "Gustus", + "letter": "s", + "count": 2 + }, + { + "word": "n-dimensional", + "letter": "n", + "count": 3 + }, + { + "word": "unvindictiveness", + "letter": "u", + "count": 1 + }, + { + "word": "corneosclerotic", + "letter": "r", + "count": 2 + }, + { + "word": "hogreeve", + "letter": "e", + "count": 3 + }, + { + "word": "hylo-", + "letter": "l", + "count": 1 + }, + { + "word": "progenerate", + "letter": "r", + "count": 2 + }, + { + "word": "phlebotomisation", + "letter": "o", + "count": 3 + }, + { + "word": "palmitin", + "letter": "n", + "count": 1 + }, + { + "word": "sebific", + "letter": "i", + "count": 2 + }, + { + "word": "extradepartmental", + "letter": "a", + "count": 3 + }, + { + "word": "multifarious", + "letter": "a", + "count": 1 + }, + { + "word": "postoptic", + "letter": "o", + "count": 2 + }, + { + "word": "foredoomed", + "letter": "o", + "count": 3 + }, + { + "word": "spasmolytic", + "letter": "m", + "count": 1 + }, + { + "word": "Pro-hungarian", + "letter": "n", + "count": 2 + }, + { + "word": "screenwriter", + "letter": "e", + "count": 3 + }, + { + "word": "bedfast", + "letter": "s", + "count": 1 + }, + { + "word": "incremented", + "letter": "n", + "count": 2 + }, + { + "word": "preorder", + "letter": "r", + "count": 3 + }, + { + "word": "countless", + "letter": "o", + "count": 1 + }, + { + "word": "simperingly", + "letter": "i", + "count": 2 + }, + { + "word": "precyclonic", + "letter": "c", + "count": 3 + }, + { + "word": "Rhodothece", + "letter": "c", + "count": 1 + }, + { + "word": "palmula", + "letter": "l", + "count": 2 + }, + { + "word": "DiBiasi", + "letter": "i", + "count": 3 + }, + { + "word": "self-rule", + "letter": "u", + "count": 1 + }, + { + "word": "electrochronometric", + "letter": "t", + "count": 2 + }, + { + "word": "fissiparity", + "letter": "i", + "count": 3 + }, + { + "word": "overimitate", + "letter": "m", + "count": 1 + }, + { + "word": "conformer", + "letter": "r", + "count": 2 + }, + { + "word": "teedle", + "letter": "e", + "count": 3 + }, + { + "word": "unsummable", + "letter": "e", + "count": 1 + }, + { + "word": "fosslfying", + "letter": "s", + "count": 2 + }, + { + "word": "defectless", + "letter": "e", + "count": 3 + }, + { + "word": "hydroquinine", + "letter": "e", + "count": 1 + }, + { + "word": "prayerful", + "letter": "r", + "count": 2 + }, + { + "word": "brilliantined", + "letter": "i", + "count": 3 + }, + { + "word": "sottie", + "letter": "s", + "count": 1 + }, + { + "word": "Gillespie", + "letter": "l", + "count": 2 + }, + { + "word": "cahincic", + "letter": "c", + "count": 3 + }, + { + "word": "Gagnon", + "letter": "o", + "count": 1 + }, + { + "word": "fundamentalist", + "letter": "t", + "count": 2 + }, + { + "word": "nonpasserine", + "letter": "n", + "count": 3 + }, + { + "word": "promittor", + "letter": "m", + "count": 1 + }, + { + "word": "quidditative", + "letter": "d", + "count": 2 + }, + { + "word": "agrostological", + "letter": "o", + "count": 3 + }, + { + "word": "cheilotomy", + "letter": "i", + "count": 1 + }, + { + "word": "gombeen-man", + "letter": "n", + "count": 2 + }, + { + "word": "oversmoothly", + "letter": "o", + "count": 3 + }, + { + "word": "Kunstlieder", + "letter": "u", + "count": 1 + }, + { + "word": "trikaya", + "letter": "a", + "count": 2 + }, + { + "word": "well-left", + "letter": "l", + "count": 3 + }, + { + "word": "allosyndesis", + "letter": "e", + "count": 1 + }, + { + "word": "lumbermill", + "letter": "m", + "count": 2 + }, + { + "word": "false-hearted", + "letter": "e", + "count": 3 + }, + { + "word": "amorality", + "letter": "r", + "count": 1 + }, + { + "word": "toolmakers", + "letter": "o", + "count": 2 + }, + { + "word": "liverleaves", + "letter": "e", + "count": 3 + }, + { + "word": "alodia", + "letter": "o", + "count": 1 + }, + { + "word": "pennaceous", + "letter": "e", + "count": 2 + }, + { + "word": "underlabourer", + "letter": "r", + "count": 3 + }, + { + "word": "besteer", + "letter": "s", + "count": 1 + }, + { + "word": "smooth-skinned", + "letter": "s", + "count": 2 + }, + { + "word": "pseudoneuropterous", + "letter": "e", + "count": 3 + }, + { + "word": "uninundated", + "letter": "t", + "count": 1 + }, + { + "word": "semirevolutionary", + "letter": "e", + "count": 2 + }, + { + "word": "sunshine-showery", + "letter": "s", + "count": 3 + }, + { + "word": "achromatic", + "letter": "o", + "count": 1 + }, + { + "word": "tatuasu", + "letter": "t", + "count": 2 + }, + { + "word": "vivisectorium", + "letter": "i", + "count": 3 + }, + { + "word": "intentionality", + "letter": "o", + "count": 1 + }, + { + "word": "Buddha", + "letter": "d", + "count": 2 + }, + { + "word": "tumulous", + "letter": "u", + "count": 3 + }, + { + "word": "gentleman-rider", + "letter": "t", + "count": 1 + }, + { + "word": "Jeanine", + "letter": "n", + "count": 2 + }, + { + "word": "submergement", + "letter": "e", + "count": 3 + }, + { + "word": "secondi", + "letter": "i", + "count": 1 + }, + { + "word": "sighingly", + "letter": "i", + "count": 2 + }, + { + "word": "physicists", + "letter": "s", + "count": 3 + }, + { + "word": "polythalamous", + "letter": "m", + "count": 1 + }, + { + "word": "flacherie", + "letter": "e", + "count": 2 + }, + { + "word": "monopotassium", + "letter": "o", + "count": 3 + }, + { + "word": "Tsitsihar", + "letter": "a", + "count": 1 + }, + { + "word": "collateralizing", + "letter": "i", + "count": 2 + }, + { + "word": "corticosteroids", + "letter": "o", + "count": 3 + }, + { + "word": "catercornerways", + "letter": "w", + "count": 1 + }, + { + "word": "fervorlessness", + "letter": "r", + "count": 2 + }, + { + "word": "dramatically", + "letter": "a", + "count": 3 + }, + { + "word": "ragamuffinism", + "letter": "s", + "count": 1 + }, + { + "word": "frondiferous", + "letter": "f", + "count": 2 + }, + { + "word": "nonperpetration", + "letter": "n", + "count": 3 + }, + { + "word": "WRESAT", + "letter": "E", + "count": 1 + }, + { + "word": "Montesco", + "letter": "o", + "count": 2 + }, + { + "word": "ethnobotanist", + "letter": "t", + "count": 3 + }, + { + "word": "pseudopodium", + "letter": "s", + "count": 1 + }, + { + "word": "hammerbird", + "letter": "m", + "count": 2 + }, + { + "word": "Marcionitism", + "letter": "i", + "count": 3 + }, + { + "word": "Curare", + "letter": "C", + "count": 1 + }, + { + "word": "Jeffreys", + "letter": "e", + "count": 2 + }, + { + "word": "allylic", + "letter": "l", + "count": 3 + }, + { + "word": "befleaing", + "letter": "l", + "count": 1 + }, + { + "word": "unwedge", + "letter": "e", + "count": 2 + }, + { + "word": "second-feet", + "letter": "e", + "count": 3 + }, + { + "word": "aspersoria", + "letter": "e", + "count": 1 + }, + { + "word": "stumpages", + "letter": "s", + "count": 2 + }, + { + "word": "bocaccio", + "letter": "c", + "count": 3 + }, + { + "word": "tram-borne", + "letter": "n", + "count": 1 + }, + { + "word": "unclogging", + "letter": "n", + "count": 2 + }, + { + "word": "crestless", + "letter": "s", + "count": 3 + }, + { + "word": "unnocturnally", + "letter": "a", + "count": 1 + }, + { + "word": "well-superintended", + "letter": "n", + "count": 2 + }, + { + "word": "unexaggerative", + "letter": "e", + "count": 3 + }, + { + "word": "anility", + "letter": "y", + "count": 1 + }, + { + "word": "ignigenous", + "letter": "n", + "count": 2 + }, + { + "word": "unbribably", + "letter": "b", + "count": 3 + }, + { + "word": "dhabb", + "letter": "a", + "count": 1 + }, + { + "word": "nonparadoxical", + "letter": "o", + "count": 2 + }, + { + "word": "disanimation", + "letter": "i", + "count": 3 + }, + { + "word": "Drugi", + "letter": "D", + "count": 1 + }, + { + "word": "noncoalescent", + "letter": "o", + "count": 2 + }, + { + "word": "radiosterilized", + "letter": "i", + "count": 3 + }, + { + "word": "pneumographic", + "letter": "h", + "count": 1 + }, + { + "word": "jellab", + "letter": "l", + "count": 2 + }, + { + "word": "infirmaries", + "letter": "i", + "count": 3 + }, + { + "word": "preconsultation", + "letter": "u", + "count": 1 + }, + { + "word": "antigens", + "letter": "n", + "count": 2 + }, + { + "word": "Teesside", + "letter": "e", + "count": 3 + }, + { + "word": "actinoelectric", + "letter": "a", + "count": 1 + }, + { + "word": "levigate", + "letter": "e", + "count": 2 + }, + { + "word": "fagottist", + "letter": "t", + "count": 3 + }, + { + "word": "colonopexy", + "letter": "y", + "count": 1 + }, + { + "word": "trioses", + "letter": "s", + "count": 2 + }, + { + "word": "mittatur", + "letter": "t", + "count": 3 + }, + { + "word": "curdwort", + "letter": "u", + "count": 1 + }, + { + "word": "sketchist", + "letter": "s", + "count": 2 + }, + { + "word": "ten-toothed", + "letter": "t", + "count": 3 + }, + { + "word": "Zobias", + "letter": "Z", + "count": 1 + }, + { + "word": "redepreciating", + "letter": "i", + "count": 2 + }, + { + "word": "overeye", + "letter": "e", + "count": 3 + }, + { + "word": "Mindererus", + "letter": "n", + "count": 1 + }, + { + "word": "thunderstrike", + "letter": "r", + "count": 2 + }, + { + "word": "infinitude", + "letter": "i", + "count": 3 + }, + { + "word": "unbudgeable", + "letter": "g", + "count": 1 + }, + { + "word": "deeny", + "letter": "e", + "count": 2 + }, + { + "word": "diminishableness", + "letter": "i", + "count": 3 + }, + { + "word": "Mawr", + "letter": "w", + "count": 1 + }, + { + "word": "reptiliform", + "letter": "r", + "count": 2 + }, + { + "word": "prelawfully", + "letter": "l", + "count": 3 + }, + { + "word": "livres", + "letter": "v", + "count": 1 + }, + { + "word": "mulled", + "letter": "l", + "count": 2 + }, + { + "word": "nonverminousness", + "letter": "s", + "count": 3 + }, + { + "word": "subclavioaxillary", + "letter": "r", + "count": 1 + }, + { + "word": "serenades", + "letter": "s", + "count": 2 + }, + { + "word": "unhumidified", + "letter": "i", + "count": 3 + }, + { + "word": "Enterolobium", + "letter": "l", + "count": 1 + }, + { + "word": "Origenist", + "letter": "i", + "count": 2 + }, + { + "word": "traveler's-tree", + "letter": "r", + "count": 3 + }, + { + "word": "phantomlike", + "letter": "l", + "count": 1 + }, + { + "word": "martyrisation", + "letter": "i", + "count": 2 + }, + { + "word": "entrepreneur's", + "letter": "r", + "count": 3 + }, + { + "word": "quirksome", + "letter": "e", + "count": 1 + }, + { + "word": "gingilis", + "letter": "g", + "count": 2 + }, + { + "word": "flimflamming", + "letter": "m", + "count": 3 + }, + { + "word": "saucing", + "letter": "c", + "count": 1 + }, + { + "word": "axoplasm", + "letter": "a", + "count": 2 + }, + { + "word": "homotaxeous", + "letter": "o", + "count": 3 + }, + { + "word": "autoloaders", + "letter": "l", + "count": 1 + }, + { + "word": "unstandard", + "letter": "a", + "count": 2 + }, + { + "word": "sorboses", + "letter": "s", + "count": 3 + }, + { + "word": "clatterer", + "letter": "c", + "count": 1 + }, + { + "word": "sagger", + "letter": "g", + "count": 2 + }, + { + "word": "ginglymostomoid", + "letter": "o", + "count": 3 + }, + { + "word": "nucleolini", + "letter": "c", + "count": 1 + }, + { + "word": "lamsiekte", + "letter": "e", + "count": 2 + }, + { + "word": "vexatiousness", + "letter": "s", + "count": 3 + }, + { + "word": "Warrensburg", + "letter": "s", + "count": 1 + }, + { + "word": "impactive", + "letter": "i", + "count": 2 + }, + { + "word": "high-spiritedness", + "letter": "i", + "count": 3 + }, + { + "word": "Llandaff", + "letter": "n", + "count": 1 + }, + { + "word": "excogitate", + "letter": "t", + "count": 2 + }, + { + "word": "I.R.A.", + "letter": ".", + "count": 3 + }, + { + "word": "swiftest", + "letter": "w", + "count": 1 + }, + { + "word": "Independista", + "letter": "n", + "count": 2 + }, + { + "word": "teleostomous", + "letter": "o", + "count": 3 + }, + { + "word": "hexametric", + "letter": "i", + "count": 1 + }, + { + "word": "reree", + "letter": "r", + "count": 2 + }, + { + "word": "fickle-mindedness", + "letter": "e", + "count": 3 + }, + { + "word": "cross-fertilize", + "letter": "t", + "count": 1 + }, + { + "word": "reposes", + "letter": "s", + "count": 2 + }, + { + "word": "pluricipital", + "letter": "i", + "count": 3 + }, + { + "word": "propinquous", + "letter": "r", + "count": 1 + }, + { + "word": "reconfigurable", + "letter": "e", + "count": 2 + }, + { + "word": "nondilatability", + "letter": "i", + "count": 3 + }, + { + "word": "poculary", + "letter": "y", + "count": 1 + }, + { + "word": "saccomyoidean", + "letter": "c", + "count": 2 + }, + { + "word": "loudishness", + "letter": "s", + "count": 3 + }, + { + "word": "xanthoxenite", + "letter": "i", + "count": 1 + }, + { + "word": "riverhood", + "letter": "o", + "count": 2 + }, + { + "word": "nonliving", + "letter": "n", + "count": 3 + }, + { + "word": "onehood", + "letter": "n", + "count": 1 + }, + { + "word": "explorator", + "letter": "o", + "count": 2 + }, + { + "word": "branniness", + "letter": "n", + "count": 3 + }, + { + "word": "hydroxylize", + "letter": "d", + "count": 1 + }, + { + "word": "willow-shaded", + "letter": "d", + "count": 2 + }, + { + "word": "trophospongial", + "letter": "o", + "count": 3 + }, + { + "word": "pomegranates", + "letter": "s", + "count": 1 + }, + { + "word": "reintroduces", + "letter": "e", + "count": 2 + }, + { + "word": "Planada", + "letter": "a", + "count": 3 + }, + { + "word": "Lucite", + "letter": "t", + "count": 1 + }, + { + "word": "Kumagai", + "letter": "a", + "count": 2 + }, + { + "word": "querulousness", + "letter": "s", + "count": 3 + }, + { + "word": "Doeg", + "letter": "o", + "count": 1 + }, + { + "word": "metaphysicians", + "letter": "s", + "count": 2 + }, + { + "word": "emblematise", + "letter": "e", + "count": 3 + }, + { + "word": "Cleone", + "letter": "o", + "count": 1 + }, + { + "word": "Semi-pythagorean", + "letter": "e", + "count": 2 + }, + { + "word": "laurustinus", + "letter": "u", + "count": 3 + }, + { + "word": "prioral", + "letter": "p", + "count": 1 + }, + { + "word": "chloroplast's", + "letter": "s", + "count": 2 + }, + { + "word": "paraphrase", + "letter": "a", + "count": 3 + }, + { + "word": "heretoch", + "letter": "c", + "count": 1 + }, + { + "word": "arenarious", + "letter": "r", + "count": 2 + }, + { + "word": "azophenetole", + "letter": "e", + "count": 3 + }, + { + "word": "prepack", + "letter": "c", + "count": 1 + }, + { + "word": "potestative", + "letter": "e", + "count": 2 + }, + { + "word": "intransmutability", + "letter": "i", + "count": 3 + }, + { + "word": "philosophized", + "letter": "e", + "count": 1 + }, + { + "word": "gigabyte", + "letter": "g", + "count": 2 + }, + { + "word": "subdolichocephalous", + "letter": "o", + "count": 3 + }, + { + "word": "mixtilion", + "letter": "l", + "count": 1 + }, + { + "word": "unesoteric", + "letter": "e", + "count": 2 + }, + { + "word": "electress", + "letter": "e", + "count": 3 + }, + { + "word": "Daniele", + "letter": "n", + "count": 1 + }, + { + "word": "coadjuvate", + "letter": "a", + "count": 2 + }, + { + "word": "multicrystalline", + "letter": "l", + "count": 3 + }, + { + "word": "machinize", + "letter": "m", + "count": 1 + }, + { + "word": "danseusse", + "letter": "e", + "count": 2 + }, + { + "word": "crossbreeded", + "letter": "e", + "count": 3 + }, + { + "word": "glummest", + "letter": "t", + "count": 1 + }, + { + "word": "Derrek", + "letter": "r", + "count": 2 + }, + { + "word": "antipestilent", + "letter": "t", + "count": 3 + }, + { + "word": "prosacral", + "letter": "s", + "count": 1 + }, + { + "word": "nationalize", + "letter": "i", + "count": 2 + }, + { + "word": "insuppressive", + "letter": "s", + "count": 3 + }, + { + "word": "yearlies", + "letter": "y", + "count": 1 + }, + { + "word": "synodalist", + "letter": "s", + "count": 2 + }, + { + "word": "merosomatous", + "letter": "o", + "count": 3 + }, + { + "word": "vertebrae", + "letter": "t", + "count": 1 + }, + { + "word": "taints", + "letter": "t", + "count": 2 + }, + { + "word": "implemented", + "letter": "e", + "count": 3 + }, + { + "word": "scofflaws", + "letter": "o", + "count": 1 + }, + { + "word": "umbellule", + "letter": "e", + "count": 2 + }, + { + "word": "reticulatogranulate", + "letter": "a", + "count": 3 + }, + { + "word": "pseudoderm", + "letter": "s", + "count": 1 + }, + { + "word": "unliteral", + "letter": "l", + "count": 2 + }, + { + "word": "photochrome", + "letter": "o", + "count": 3 + }, + { + "word": "renounceable", + "letter": "l", + "count": 1 + }, + { + "word": "capriciously", + "letter": "c", + "count": 2 + }, + { + "word": "wigwagger", + "letter": "g", + "count": 3 + }, + { + "word": "godlike", + "letter": "i", + "count": 1 + }, + { + "word": "cardplaying", + "letter": "a", + "count": 2 + }, + { + "word": "electrooculogram", + "letter": "o", + "count": 3 + }, + { + "word": "superintend", + "letter": "t", + "count": 1 + }, + { + "word": "regulatress", + "letter": "r", + "count": 2 + }, + { + "word": "replacement", + "letter": "e", + "count": 3 + }, + { + "word": "scaddle", + "letter": "e", + "count": 1 + }, + { + "word": "merging", + "letter": "g", + "count": 2 + }, + { + "word": "invaccination", + "letter": "n", + "count": 3 + }, + { + "word": "pterygopalatine", + "letter": "r", + "count": 1 + }, + { + "word": "sweety", + "letter": "e", + "count": 2 + }, + { + "word": "intersexualities", + "letter": "i", + "count": 3 + }, + { + "word": "lovability", + "letter": "o", + "count": 1 + }, + { + "word": "victim's", + "letter": "i", + "count": 2 + }, + { + "word": "heterolateral", + "letter": "e", + "count": 3 + }, + { + "word": "cowleeching", + "letter": "i", + "count": 1 + }, + { + "word": "Megaceros", + "letter": "e", + "count": 2 + }, + { + "word": "tauromaquia", + "letter": "a", + "count": 3 + }, + { + "word": "award", + "letter": "d", + "count": 1 + }, + { + "word": "oval-leaved", + "letter": "l", + "count": 2 + }, + { + "word": "nomologies", + "letter": "o", + "count": 3 + }, + { + "word": "pulvini", + "letter": "p", + "count": 1 + }, + { + "word": "semicontraction", + "letter": "o", + "count": 2 + }, + { + "word": "spikedness", + "letter": "s", + "count": 3 + }, + { + "word": "foghorn", + "letter": "h", + "count": 1 + }, + { + "word": "Mlaga", + "letter": "a", + "count": 2 + }, + { + "word": "alicyclic", + "letter": "c", + "count": 3 + }, + { + "word": "salinometry", + "letter": "n", + "count": 1 + }, + { + "word": "Maccarone", + "letter": "a", + "count": 2 + }, + { + "word": "electrosmosis", + "letter": "s", + "count": 3 + }, + { + "word": "manhaden", + "letter": "d", + "count": 1 + }, + { + "word": "emotionalizing", + "letter": "o", + "count": 2 + }, + { + "word": "unaddicted", + "letter": "d", + "count": 3 + }, + { + "word": "bequalm", + "letter": "u", + "count": 1 + }, + { + "word": "world-redeeming", + "letter": "r", + "count": 2 + }, + { + "word": "giggles", + "letter": "g", + "count": 3 + }, + { + "word": "month", + "letter": "m", + "count": 1 + }, + { + "word": "rotten-hearted", + "letter": "r", + "count": 2 + }, + { + "word": "unpreciseness", + "letter": "s", + "count": 3 + }, + { + "word": "trysail", + "letter": "l", + "count": 1 + }, + { + "word": "tricentenary", + "letter": "n", + "count": 2 + }, + { + "word": "publicities", + "letter": "i", + "count": 3 + }, + { + "word": "skeldock", + "letter": "e", + "count": 1 + }, + { + "word": "Hebrician", + "letter": "i", + "count": 2 + }, + { + "word": "commem", + "letter": "m", + "count": 3 + }, + { + "word": "synodist", + "letter": "y", + "count": 1 + }, + { + "word": "platan", + "letter": "a", + "count": 2 + }, + { + "word": "singultuses", + "letter": "s", + "count": 3 + }, + { + "word": "chaffweed", + "letter": "a", + "count": 1 + }, + { + "word": "trachymedusan", + "letter": "a", + "count": 2 + }, + { + "word": "indeterminableness", + "letter": "n", + "count": 3 + }, + { + "word": "geologer", + "letter": "r", + "count": 1 + }, + { + "word": "elderwood", + "letter": "d", + "count": 2 + }, + { + "word": "cacocholia", + "letter": "c", + "count": 3 + }, + { + "word": "unneglectful", + "letter": "g", + "count": 1 + }, + { + "word": "redrugged", + "letter": "r", + "count": 2 + }, + { + "word": "tempest-rocked", + "letter": "e", + "count": 3 + }, + { + "word": "burdensomeness", + "letter": "b", + "count": 1 + }, + { + "word": "Onopordon", + "letter": "n", + "count": 2 + }, + { + "word": "horrorize", + "letter": "r", + "count": 3 + }, + { + "word": "lushest", + "letter": "h", + "count": 1 + }, + { + "word": "conjugator", + "letter": "o", + "count": 2 + }, + { + "word": "Paragonah", + "letter": "a", + "count": 3 + }, + { + "word": "enaliosaur", + "letter": "i", + "count": 1 + }, + { + "word": "unpalliable", + "letter": "a", + "count": 2 + }, + { + "word": "nonlimiting", + "letter": "i", + "count": 3 + }, + { + "word": "outgarth", + "letter": "r", + "count": 1 + }, + { + "word": "explanting", + "letter": "n", + "count": 2 + }, + { + "word": "cerebriformly", + "letter": "r", + "count": 3 + }, + { + "word": "raisings", + "letter": "n", + "count": 1 + }, + { + "word": "bastardizing", + "letter": "a", + "count": 2 + }, + { + "word": "emblematize", + "letter": "e", + "count": 3 + }, + { + "word": "kilning", + "letter": "l", + "count": 1 + }, + { + "word": "Astolat", + "letter": "t", + "count": 2 + }, + { + "word": "microprogrammed", + "letter": "r", + "count": 3 + }, + { + "word": "Royden", + "letter": "d", + "count": 1 + }, + { + "word": "unparriable", + "letter": "a", + "count": 2 + }, + { + "word": "nonlinguistic", + "letter": "i", + "count": 3 + }, + { + "word": "inharmoniousness", + "letter": "h", + "count": 1 + }, + { + "word": "postgastric", + "letter": "s", + "count": 2 + }, + { + "word": "unblenching", + "letter": "n", + "count": 3 + }, + { + "word": "gill-like", + "letter": "-", + "count": 1 + }, + { + "word": "patternlike", + "letter": "e", + "count": 2 + }, + { + "word": "photosynthometer", + "letter": "o", + "count": 3 + }, + { + "word": "undermined", + "letter": "i", + "count": 1 + }, + { + "word": "antiweed", + "letter": "e", + "count": 2 + }, + { + "word": "hospitableness", + "letter": "s", + "count": 3 + }, + { + "word": "glaga", + "letter": "l", + "count": 1 + }, + { + "word": "cerolite", + "letter": "e", + "count": 2 + }, + { + "word": "ultrafiltrate", + "letter": "t", + "count": 3 + }, + { + "word": "night-veiled", + "letter": "d", + "count": 1 + }, + { + "word": "noncompressible", + "letter": "e", + "count": 2 + }, + { + "word": "twenty-payment", + "letter": "t", + "count": 3 + }, + { + "word": "astay", + "letter": "s", + "count": 1 + }, + { + "word": "Meggs", + "letter": "g", + "count": 2 + }, + { + "word": "careeners", + "letter": "e", + "count": 3 + }, + { + "word": "Arunta", + "letter": "n", + "count": 1 + }, + { + "word": "bajree", + "letter": "e", + "count": 2 + }, + { + "word": "irreligious", + "letter": "i", + "count": 3 + }, + { + "word": "Vereine", + "letter": "i", + "count": 1 + }, + { + "word": "percipi", + "letter": "i", + "count": 2 + }, + { + "word": "omnibus-driving", + "letter": "i", + "count": 3 + }, + { + "word": "rededuct", + "letter": "c", + "count": 1 + }, + { + "word": "craniopharyngeal", + "letter": "n", + "count": 2 + }, + { + "word": "mammocking", + "letter": "m", + "count": 3 + }, + { + "word": "goldang", + "letter": "d", + "count": 1 + }, + { + "word": "concentrators", + "letter": "o", + "count": 2 + }, + { + "word": "elongato-conical", + "letter": "o", + "count": 3 + }, + { + "word": "munificence", + "letter": "u", + "count": 1 + }, + { + "word": "lasagnas", + "letter": "s", + "count": 2 + }, + { + "word": "bookworm", + "letter": "o", + "count": 3 + }, + { + "word": "Tereus", + "letter": "r", + "count": 1 + }, + { + "word": "plexal", + "letter": "l", + "count": 2 + }, + { + "word": "yellow-lit", + "letter": "l", + "count": 3 + }, + { + "word": "Hedelman", + "letter": "H", + "count": 1 + }, + { + "word": "Neotropical", + "letter": "o", + "count": 2 + }, + { + "word": "pseudogentlemanly", + "letter": "e", + "count": 3 + }, + { + "word": "Nidulariales", + "letter": "s", + "count": 1 + }, + { + "word": "stereostatic", + "letter": "e", + "count": 2 + }, + { + "word": "antilitter", + "letter": "t", + "count": 3 + }, + { + "word": "seriation", + "letter": "a", + "count": 1 + }, + { + "word": "recapturing", + "letter": "r", + "count": 2 + }, + { + "word": "Ledbetter", + "letter": "e", + "count": 3 + }, + { + "word": "pullen", + "letter": "u", + "count": 1 + }, + { + "word": "finnac", + "letter": "n", + "count": 2 + }, + { + "word": "microclimatological", + "letter": "l", + "count": 3 + }, + { + "word": "monoblepsis", + "letter": "n", + "count": 1 + }, + { + "word": "semifloating", + "letter": "i", + "count": 2 + }, + { + "word": "warm-tempered", + "letter": "e", + "count": 3 + }, + { + "word": "shakespeareans", + "letter": "r", + "count": 1 + }, + { + "word": "preventiveness", + "letter": "s", + "count": 2 + }, + { + "word": "eyewitness", + "letter": "e", + "count": 3 + }, + { + "word": "weather-bitten", + "letter": "h", + "count": 1 + }, + { + "word": "occipitohyoid", + "letter": "c", + "count": 2 + }, + { + "word": "quinicine", + "letter": "i", + "count": 3 + }, + { + "word": "crunchiness", + "letter": "i", + "count": 1 + }, + { + "word": "peregrinations", + "letter": "r", + "count": 2 + }, + { + "word": "fire-retardant", + "letter": "r", + "count": 3 + }, + { + "word": "villanage", + "letter": "v", + "count": 1 + }, + { + "word": "rhomboquadratic", + "letter": "o", + "count": 2 + }, + { + "word": "superparliamentary", + "letter": "a", + "count": 3 + }, + { + "word": "helenn", + "letter": "h", + "count": 1 + }, + { + "word": "raveled", + "letter": "e", + "count": 2 + }, + { + "word": "pseudoeconomical", + "letter": "o", + "count": 3 + }, + { + "word": "outbakes", + "letter": "u", + "count": 1 + }, + { + "word": "unmethodising", + "letter": "i", + "count": 2 + }, + { + "word": "totalist", + "letter": "t", + "count": 3 + }, + { + "word": "Capricorni", + "letter": "n", + "count": 1 + }, + { + "word": "spinobulbar", + "letter": "b", + "count": 2 + }, + { + "word": "enterotoxication", + "letter": "o", + "count": 3 + }, + { + "word": "subdilated", + "letter": "u", + "count": 1 + }, + { + "word": "Swanhildas", + "letter": "a", + "count": 2 + }, + { + "word": "respirometry", + "letter": "r", + "count": 3 + }, + { + "word": "worse-served", + "letter": "o", + "count": 1 + }, + { + "word": "hypoalimentation", + "letter": "i", + "count": 2 + }, + { + "word": "unyieldingness", + "letter": "n", + "count": 3 + }, + { + "word": "prestandardization", + "letter": "o", + "count": 1 + }, + { + "word": "etheling", + "letter": "e", + "count": 2 + }, + { + "word": "splanchnodynia", + "letter": "n", + "count": 3 + }, + { + "word": "move", + "letter": "e", + "count": 1 + }, + { + "word": "paperback's", + "letter": "a", + "count": 2 + }, + { + "word": "diverse-natured", + "letter": "e", + "count": 3 + }, + { + "word": "seignior", + "letter": "n", + "count": 1 + }, + { + "word": "average", + "letter": "a", + "count": 2 + }, + { + "word": "catenarian", + "letter": "a", + "count": 3 + }, + { + "word": "gumptious", + "letter": "m", + "count": 1 + }, + { + "word": "white-painted", + "letter": "e", + "count": 2 + }, + { + "word": "Nekhebet", + "letter": "e", + "count": 3 + }, + { + "word": "airframes", + "letter": "m", + "count": 1 + }, + { + "word": "pollin-", + "letter": "l", + "count": 2 + }, + { + "word": "Grandmontine", + "letter": "n", + "count": 3 + }, + { + "word": "mecurialism", + "letter": "e", + "count": 1 + } +] \ No newline at end of file diff --git a/pipelinerl/entrypoints/environment.py b/pipelinerl/entrypoints/environment.py new file mode 100644 index 0000000..d5e05d2 --- /dev/null +++ b/pipelinerl/entrypoints/environment.py @@ -0,0 +1,17 @@ +import hydra +from omegaconf import DictConfig + +from pipelinerl.utils import better_crashing + + +@hydra.main(config_path="../../conf", config_name="base", version_base="1.3.2") +def hydra_entrypoint(cfg: DictConfig): + with better_crashing("environment"): + environment = hydra.utils.instantiate(cfg.environment) + this_job, = [job for job in cfg.jobs if job["idx"] == cfg.me.job_idx] + port = this_job["port"] + environment.launch(port=port) + + +if __name__ == "__main__": + hydra_entrypoint() diff --git a/pipelinerl/entrypoints/verifier.py b/pipelinerl/entrypoints/verifier.py deleted file mode 100644 index 89b4d53..0000000 --- a/pipelinerl/entrypoints/verifier.py +++ /dev/null @@ -1,15 +0,0 @@ -import hydra -from omegaconf import DictConfig - -from pipelinerl.verifier_api import run_verifier -from pipelinerl.utils import better_crashing - - -@hydra.main(config_path="../../conf", config_name="rl_deepseek_async", version_base="1.3.2") -def hydra_entrypoint(cfg: DictConfig): - with better_crashing("actor"): - run_verifier(cfg) - - -if __name__ == "__main__": - hydra_entrypoint() diff --git a/pipelinerl/launch.py b/pipelinerl/launch.py index cbcf752..19f515c 100644 --- a/pipelinerl/launch.py +++ b/pipelinerl/launch.py @@ -14,7 +14,7 @@ from pipelinerl.state import TrainerState from pipelinerl.streams import SingleStreamSpec, connect_to_redis, read_stream, set_streams_backend, write_to_streams from pipelinerl.utils import terminate_with_children -from pipelinerl.world import WorldMap +from pipelinerl.world import Job, WorldMap logger = logging.getLogger(__name__) @@ -55,7 +55,7 @@ def run_ref_llm(cfg: DictConfig, preprocessor_llm_idx: int, local_idx: int, gpus kwargs = cfg.vllm_config.vllm_kwargs if kwargs["num-scheduler-steps"] > 1: kwargs["num-scheduler-steps"] = 1 - logger.warning(f"Set num-scheduler-steps to 1 for reference vLLM") + logger.warning("Set num-scheduler-steps to 1 for reference vLLM") log_dir = exp_dir / f"ref_vllm_{preprocessor_llm_idx}" os.makedirs(log_dir, exist_ok=True) @@ -81,8 +81,8 @@ def run_ref_llm(cfg: DictConfig, preprocessor_llm_idx: int, local_idx: int, gpus gpu_str = ",".join([str(gpu) for gpu in gpus]) logger.info(f"Running reference LLM with command: {' '.join(cmd)} with gpus: {gpu_str}") - log_file_path = os.path.join(log_dir, f"stdout.log") - err_file_path = os.path.join(log_dir, f"stderr.log") + log_file_path = os.path.join(log_dir, "stdout.log") + err_file_path = os.path.join(log_dir, "stderr.log") with open(log_file_path, "a") as log_file, open(err_file_path, "a") as err_file: yield _popen( cmd, @@ -138,8 +138,9 @@ def run_actor_llm( gpu_str = ",".join([str(gpu) for gpu in gpus]) logger.info(f"Running actor_llm with command: {' '.join(cmd)} on gpus: {gpu_str}") - log_file_path = os.path.join(log_dir, f"stdout.log") - err_file_path = os.path.join(log_dir, f"stderr.log") + save_command(log_dir, cmd) + log_file_path = os.path.join(log_dir, "stdout.log") + err_file_path = os.path.join(log_dir, "stderr.log") with open(log_file_path, "a") as log_file, open(err_file_path, "a") as err_file: yield _popen( cmd, @@ -166,30 +167,33 @@ def run_actor(world_map: WorldMap, actor_idx: int, exp_dir: Path): f"+me.llm_urls={llm_urls}", ] logger.info(f"Running actor with command: {' '.join(cmd)}") + save_command(exp_dir / "actor", cmd) yield _popen( cmd, env=dict(os.environ), ) -def run_verifier(cfg: DictConfig): +def run_environment(cfg: DictConfig, job: Job): # run in a subprocess like in the rest of the code + run_dir = Path(cfg.output_dir) / f"environment_{job.replica_idx}" cmd = [ "python", "-m", - "pipelinerl.entrypoints.verifier", + "pipelinerl.entrypoints.environment", "--config-dir", f"{cfg.output_dir}/conf", "--config-name", "exp_config", f"output_dir={cfg.output_dir}", - f"hydra.run.dir={cfg.output_dir}/verifier", + f"hydra.run.dir={str(run_dir)}", + f"me.job_idx={job.idx}", ] - logger.info(f"Running verifier with command: {' '.join(cmd)}") - log_dir = os.path.join(cfg.output_dir, "verifier") - os.makedirs(log_dir, exist_ok=True) - log_file_path = os.path.join(log_dir, f"stdout.log") - err_file_path = os.path.join(log_dir, f"stderr.log") + logger.info(f"Running environment with command: {' '.join(cmd)}") + os.makedirs(run_dir, exist_ok=True) + save_command(run_dir, cmd) + log_file_path = str(run_dir / "stdout.log") + err_file_path = str(run_dir / "stderr.log") with open(log_file_path, "a") as log_file, open(err_file_path, "a") as err_file: yield _popen( cmd, @@ -285,6 +289,7 @@ def run_finetune(cfg: DictConfig, world_map: WorldMap, gpus: list[int], exp_dir: cmd.append("finetune.send_weight_updates=False") logger.info(f"Running finetune with command: {' '.join(cmd)}") + save_command(exp_dir / "finetune", cmd) env = dict(os.environ) env["DS_ENV_FILE"] = str(exp_dir / ".deepspeed_env") yield _popen(cmd, env=env) @@ -307,6 +312,7 @@ def run_preprocess(world_map: WorldMap, preprocessor_idx: int, exp_dir: Path): f"+me.llm_urls={llm_urls}", ] logger.info(f"Running preprocess with command: {' '.join(cmd)}") + save_command(exp_dir / "preprocess", cmd) yield _popen( cmd, env=dict(os.environ), @@ -329,9 +335,22 @@ def run_redis(cfg: DictConfig): cfg.streams.save, ] logger.info(f"Running redis with command: {' '.join(cmd)}") + save_command(Path(cfg.output_dir) / "redis", cmd) yield _popen(cmd, env=dict(os.environ)) +def save_command(script_dir: Path, cmd): + os.makedirs(script_dir, exist_ok=True) + script_path = script_dir / "start.sh" + with open(script_path, "w") as f: + f.write("#!/bin/bash\n") + # Properly quote arguments for the shell script + quoted_cmd = [f"'{arg}'" if " " in arg or "$" in arg else arg for arg in cmd] + f.write(" ".join(quoted_cmd) + "\n") + os.chmod(script_path, 0o755) + logger.info(f"Saved start script to {script_path}") + + def clean_up(exp_dir, force_restart): logger.info("Cleaning up streams directory") if os.path.exists(f"{exp_dir}/streams"): @@ -386,7 +405,7 @@ def gently_stop_all_processes(): gently_stop_all_processes() sys.exit(1) # TODO: make the watcdog code below more stable - # if (trainer_state is not Noneq + # if (trainer_state is not None # and (version := trainer_state.propagated_weight_version is not None) # and version > last_trainer_version): # last_trainer_version = version @@ -415,7 +434,7 @@ def debug_link_streams(cfg: DictConfig, topics: list[str]): def launch_jobs(cfg: DictConfig, world_map: WorldMap, job_kind_filter: list | None = None): exp_dir = Path(cfg.output_dir) processes = [] - all_job_kinds = ["actor", "verifier", "actor_llm", "preprocessor", "preprocessor_llm", "finetune"] + all_job_kinds = ["actor", "environment", "actor_llm", "preprocessor", "preprocessor_llm", "finetune"] if job_kind_filter is None: job_kind_filter = all_job_kinds for job in world_map.my_jobs(): @@ -425,8 +444,8 @@ def launch_jobs(cfg: DictConfig, world_map: WorldMap, job_kind_filter: list | No continue if job.kind == "actor": processes.extend(run_actor(world_map, job.replica_idx, exp_dir)) - elif job.kind == "verifier": - processes.extend(run_verifier(cfg)) + elif job.kind == "environment": + processes.extend(run_environment(cfg, job)) elif job.kind == "actor_llm": processes.extend(run_actor_llm(cfg, world_map, job.replica_idx, job.local_idx, job.gpus, exp_dir)) elif job.kind == "preprocessor": @@ -464,6 +483,7 @@ def main(cfg: DictConfig): log_file = exp_dir / "launcher" / f"launcher_{os.environ.get('RANK', 0)}.log" setup_logging(log_file) world_map = WorldMap(cfg, verbose=True) + cfg.jobs = [job.model_dump() for job in world_map.get_all_jobs()] group = str(exp_dir) root = cfg.finetune.wandb_workspace_root @@ -493,7 +513,7 @@ def main(cfg: DictConfig): clean_up(exp_dir, cfg.force_restart) os.makedirs(config_dir, exist_ok=True) OmegaConf.save(cfg, config_dir / "exp_config.yaml") - logger.info(f"Orchestrator 0 created the exp folder") + logger.info("Orchestrator 0 created the exp folder") if cfg.streams.backend == "redis": processes.extend(run_redis(cfg)) redis = connect_to_redis(cfg.streams) @@ -525,7 +545,7 @@ def main(cfg: DictConfig): if cfg.debug.mode == "finetune": processes.extend(launch_jobs(cfg, world_map, ["finetune"])) elif cfg.debug.mode == "actor": - processes.extend(launch_jobs(cfg, world_map, ["actor", "verifier", "actor_llm"])) + processes.extend(launch_jobs(cfg, world_map, ["actor", "environment", "actor_llm"])) elif cfg.debug.mode == "preprocessor": processes.extend(launch_jobs(cfg, world_map, ["preprocessor", "preprocessor_llm"])) elif cfg.debug.mode in ["", "open_loop"]: diff --git a/pipelinerl/load_datasets.py b/pipelinerl/math/load_datasets.py similarity index 100% rename from pipelinerl/load_datasets.py rename to pipelinerl/math/load_datasets.py diff --git a/pipelinerl/math_rollouts.py b/pipelinerl/math/rollouts.py similarity index 68% rename from pipelinerl/math_rollouts.py rename to pipelinerl/math/rollouts.py index e28d9cf..c3599e5 100644 --- a/pipelinerl/math_rollouts.py +++ b/pipelinerl/math/rollouts.py @@ -1,13 +1,17 @@ import time +import random + import aiohttp from omegaconf import DictConfig from pydantic import BaseModel -from tapeagents.core import Prompt, LLMCall, TrainingText +from pipelinerl.rollouts import RolloutResult +from pipelinerl.world import Job +from tapeagents.core import Prompt from tapeagents.llms.trainable import TrainableLLM -from pipelinerl.finetune.data import MASKED_TOKEN_ID from pipelinerl.async_llm import llm_async_generate -from pipelinerl.verifier_api import verify_answer_rpc +from pipelinerl.finetune.data import MASKED_TOKEN_ID +from pipelinerl.math.verifier_api import verify_answer_rpc class RewardTable(BaseModel): @@ -21,36 +25,41 @@ class RewardTable(BaseModel): correct_answer_finished: float -class RolloutResult(BaseModel): - training_texts: list[TrainingText] - metrics: dict[str, float] - latency: float - # optional so fields that it can be filled later after RolloutResult is created - model_version: int | None = None - dataset_name: str | None = None - group_id: str | None = None - - def make_prompt(problem: dict, cfg: DictConfig) -> Prompt: messages = [] - if cfg.system_prompt: - messages.append({"role": "system", "content": cfg.system_prompt}) - messages.append({"role": "user", "content": cfg.task_template.format(task=problem["task"])}) + if cfg.actor.system_prompt: + messages.append({"role": "system", "content": cfg.actor.system_prompt}) + messages.append({"role": "user", "content": cfg.actor.task_template.format(task=problem["task"])}) return Prompt(messages=messages) -async def process_llm_call( - session: aiohttp.ClientSession, - verifier_cfg: DictConfig, - llm_call: LLMCall, +async def generate_math_rollout( + cfg: DictConfig, llm: TrainableLLM, - answer: str, # Gold answer to verify against - rewards: RewardTable, - discount_factor: float, -) -> tuple[TrainingText, dict[str, float]]: + problem: dict, + session: aiohttp.ClientSession, +) -> RolloutResult: + prompt = make_prompt(problem=problem, cfg=cfg) + time_start = time.time() + llm_call = await llm_async_generate(llm, prompt, session) + latency = time.time() - time_start + assert llm_call.output.content is not None + rewards = RewardTable(**dict(cfg.rewards)) + discount_factor = cfg.actor.discount_factor + + # math_verify is a fast environment, no support for environment replicas for now + env_jobs = [Job(**job) for job in cfg.jobs if job["kind"] == "environment"] + # choose the job randomly + env_job = random.choice(env_jobs) + assert env_job.port is not None answer_status = await verify_answer_rpc( - session=session, verifier_cfg=verifier_cfg, prediction=llm_call.output.content, gold=answer, strict=True + session=session, + host=env_job.hostname, + port=env_job.port, + prediction=llm_call.output.content, + gold=problem["answer"], + strict=True, ) trace = llm.make_training_text(llm_call.prompt, llm_call.output) @@ -93,7 +102,7 @@ async def process_llm_call( trace.reward = reward trace.logprobs = [lp.logprob for lp in llm_call.logprobs if lp.generated] - stats = { + metrics = { "reward": reward, "success": answer_status == "correct", "no_error": answer_status != "unparsable", @@ -103,26 +112,4 @@ async def process_llm_call( "overflow": 0 if finished else 1, } - return trace, stats - - -async def generate_math_rollout( - cfg: DictConfig, - llm: TrainableLLM, - problem: dict, - session: aiohttp.ClientSession, -) -> RolloutResult: - prompt = make_prompt(problem=problem, cfg=cfg) - time_start = time.time() - llm_call = await llm_async_generate(llm, prompt, session) - latency = time.time() - time_start - sample, metrics = await process_llm_call( - session=session, - verifier_cfg=cfg.verifier, - llm_call=llm_call, - llm=llm, - answer=problem["answer"], - rewards=RewardTable(**dict(cfg.rewards)), - discount_factor=cfg.discount_factor, - ) - return RolloutResult(training_texts=[sample], metrics=metrics, latency=latency, dataset_name=problem.get("dataset")) + return RolloutResult(training_texts=[trace], metrics=metrics, latency=latency, dataset_name=problem.get("dataset")) diff --git a/pipelinerl/verifier_api.py b/pipelinerl/math/verifier_api.py similarity index 77% rename from pipelinerl/verifier_api.py rename to pipelinerl/math/verifier_api.py index a6bcc42..7b3835a 100644 --- a/pipelinerl/verifier_api.py +++ b/pipelinerl/math/verifier_api.py @@ -157,38 +157,10 @@ def verify_countdown(prediction: str, gold: str) -> str: return "wrong" -def run_verifier(cfg: DictConfig): - """ - Serve the verification API using FastAPI. - """ - app = FastAPI() - # Create a process pool with 4 workers - with ProcessPoolExecutor(max_workers=4) as process_pool: - - @app.post("/verify_answer") - async def verify(request: dict): - prediction = request["prediction"] - gold = request["gold"] - strict = request["strict"] - max_prediction_length = request["max_prediction_length"] - - # Run verification in the process pool to avoid blocking the main thread - loop = asyncio.get_event_loop() - answer_status = await loop.run_in_executor( - process_pool, partial(verify_answer, prediction, gold, strict, max_prediction_length) - ) - return JSONResponse(content={"answer_status": answer_status}) - - @app.get("/health") - async def health(): - return JSONResponse(content={"status": "ok"}) - - uvicorn.run(app, host="0.0.0.0", port=cfg.verifier.port, timeout_keep_alive=60) - - async def verify_answer_rpc( session: aiohttp.ClientSession, - verifier_cfg: DictConfig, + host: str, + port: int, prediction: str, gold: str, strict: bool = True, @@ -204,7 +176,7 @@ async def verify_answer_rpc( "max_prediction_length": max_prediction_length, } async with session.post( - f"http://{verifier_cfg.host}:{verifier_cfg.port}/verify_answer", + f"http://{host}:{port}/verify_answer", json=json, ) as response: if response.status == 200: @@ -216,16 +188,33 @@ async def verify_answer_rpc( raise ValueError("Error verifying answer") -def wait_for_verifier(verifier_cfg: DictConfig): - """ - Wait for the verifier to be ready. - """ - while True: - # use requests - try: - response = requests.get(f"http://{verifier_cfg.host}:{verifier_cfg.port}/health") - if response.status_code == 200: - break - except: - logger.info("Verifier not ready yet, waiting...") - time.sleep(5.0) +class MathEnvironment: + + def launch(self, port: int): + """ + Serve the verification API using FastAPI. + """ + app = FastAPI() + # Create a process pool with 4 workers + with ProcessPoolExecutor(max_workers=4) as process_pool: + @app.post("/verify_answer") + async def verify(request: dict): + prediction = request["prediction"] + gold = request["gold"] + strict = request["strict"] + max_prediction_length = request["max_prediction_length"] + + # Run verification in the process pool to avoid blocking the main thread + loop = asyncio.get_event_loop() + answer_status = await loop.run_in_executor( + process_pool, partial(verify_answer, prediction, gold, strict, max_prediction_length) + ) + return JSONResponse(content={"answer_status": answer_status}) + + @app.get("/health") + async def health(): + return JSONResponse(content={"status": "ok"}) + + uvicorn.run(app, host="0.0.0.0", port=port, timeout_keep_alive=60) + + diff --git a/pipelinerl/rollouts.py b/pipelinerl/rollouts.py new file mode 100644 index 0000000..cdd8ce1 --- /dev/null +++ b/pipelinerl/rollouts.py @@ -0,0 +1,31 @@ + +from pydantic import BaseModel + +from pipelinerl.finetune.data import MASKED_TOKEN_ID +from tapeagents.core import LLMCall, TrainingText +from tapeagents.llms.trainable import TrainableLLM + + +class RolloutResult(BaseModel): + training_texts: list[TrainingText] + metrics: dict[str, float] + latency: float + # optional so fields that it can be filled later after RolloutResult is created + model_version: int | None = None + dataset_name: str | None = None + group_id: str | None = None + + +def make_training_text(llm: TrainableLLM, llm_call: LLMCall) -> TrainingText: + # TODO: integrate this to TapeAgents + training_text = llm.make_training_text(llm_call.prompt, llm_call.output) + if not llm_call.logprobs: + raise ValueError("Logprobs are required to make training data for RL") + input_ids = [lp.token_id for lp in llm_call.logprobs] + labels = [lp.token_id for lp in llm_call.logprobs if lp.generated] + # Apply masking to input tokens that aren't generated + labels = [MASKED_TOKEN_ID] * (len(input_ids) - len(labels)) + labels + training_text.input_ids = input_ids + training_text.labels = labels + training_text.logprobs = [lp.logprob for lp in llm_call.logprobs if lp.generated] + return training_text \ No newline at end of file diff --git a/pipelinerl/run_actor.py b/pipelinerl/run_actor.py index 8fc7d4d..d47cf52 100644 --- a/pipelinerl/run_actor.py +++ b/pipelinerl/run_actor.py @@ -1,31 +1,27 @@ +import asyncio import logging import math -from multiprocessing.managers import SharedMemoryManager +import multiprocessing as mp import os import queue import random import time -import multiprocessing as mp from collections import defaultdict +from multiprocessing.managers import SharedMemoryManager from pathlib import Path -import uvloop import aiohttp - +import hydra +import uvloop from omegaconf import DictConfig from pydantic import BaseModel, Field - -from pipelinerl.shared_memory_array import SharedMemoryArray -from pipelinerl.verifier_api import wait_for_verifier from tapeagents.llms import TrainableLLM -from pipelinerl.finetune.logging_ import flatten_dict_config, init_wandb import wandb -from pipelinerl.load_datasets import load_datasets -from pipelinerl.math_rollouts import RolloutResult, generate_math_rollout +from pipelinerl.finetune.logging_ import flatten_dict_config, init_wandb +from pipelinerl.rollouts import RolloutResult +from pipelinerl.shared_memory_array import SharedMemoryArray from pipelinerl.state import TrainerState -import asyncio -from collections import defaultdict from pipelinerl.streams import ( SingleStreamSpec, StreamSpec, @@ -36,9 +32,10 @@ from .utils import ( always_or_never_success_stats, - calculate_stats, calculate_per_group_stats, + calculate_stats, setup_logging, + wait_for_environments, wait_for_inference_servers, ) @@ -130,6 +127,8 @@ async def schedule_rollouts( max_group_size_bytes = 0 # Track rollouts per problem group group_rollouts = {} + rollout_policy = hydra.utils.get_method(cfg.actor.rollout_policy) + logger.info(f"Use rollout policy: {rollout_policy}") async def rollout_and_maybe_produce_result( problem: dict, @@ -143,7 +142,7 @@ async def rollout_and_maybe_produce_result( llm = llms[llm_index] model_version = trainer_state.propagated_weight_version assert model_version is not None - rollout_result = await generate_math_rollout(cfg, llm, problem, session) + rollout_result = await rollout_policy(cfg, llm, problem, session) rollout_result.model_version = model_version # Make a group id that will be different from groups made by another rollout maker full_group_id = f"{scheduler_name}_{group_id}" @@ -164,7 +163,7 @@ async def rollout_and_maybe_produce_result( finished_rollouts += 1 except Exception as e: # Cancel all tasks except the current one - logger.error("Exception in rollout", exc_info=e) + logger.error("Exception in rollout, stop all other rollout tasks", exc_info=e) current_task = asyncio.current_task(loop=loop) for task in asyncio.all_tasks(loop=loop): if task != current_task: @@ -356,6 +355,7 @@ def run(self, dataset: list[tuple[str, dict]]): published_samples = 0 submitted_groups = 0 finished_groups = 0 + last_log_time = 0 expected_number_of_samples = -1 if self.is_training else len(dataset) if expected_number_of_samples > 0: logger.info(f"Will stop after {expected_number_of_samples} samples") @@ -477,22 +477,22 @@ def run(self, dataset: list[tuple[str, dict]]): # if we are training publish stats at every step else if all tapes are finished, publish stats if self.is_training or published_samples == expected_number_of_samples: if self.is_training: - loop_stats = { - "published_samples": published_samples, - "samples_in_queue": samples_in_queue, - "finished_groups": finished_groups, - "published_model_version": max_model_version, - "latency": max_latency, - "time_since_start": time.time() - loop_start_time, - } + log_time = time.monotonic() + if log_time - last_log_time > self.cfg.actor.log_each_n_secs: + loop_stats = { + "published_samples": published_samples, + "queue/problems": self.problem_queue.qsize(), + "queue/samples": samples_in_queue, + "finished_groups": finished_groups, + "published_model_version": max_model_version, + "latency": max_latency, + "time_since_start": time.time() - loop_start_time, + } + self.publish_stats(stats_writer=stats_writer, loop_stats=loop_stats, split_name=split_name) + last_log_time = log_time else: loop_stats = {"published_model_version": max_model_version} - - self.publish_stats( - stats_writer=stats_writer, - loop_stats=loop_stats, - split_name=split_name, - ) + self.publish_stats(stats_writer=stats_writer, loop_stats=loop_stats, split_name=split_name) if published_samples == expected_number_of_samples: logger.info(f"Finished {expected_number_of_samples} samples, stopping actor loop") @@ -545,13 +545,13 @@ def publish_stats(self, stats_writer: StreamWriter, loop_stats, split_name: str | {"output_tokens_" + k: v for k, v in calculate_stats(self.output_tokens[dataset_name]).items()} | {"overflows_" + k: v for k, v in calculate_stats(self.overflows[dataset_name]).items()} ) - sub_stats = {dataset_name + "_" + k: v for k, v in sub_stats.items()} + sub_stats = {dataset_name + "/" + k: v for k, v in sub_stats.items()} stats |= sub_stats stats |= loop_stats if loop_stats.get("finished_groups", 0) >= 2 * self.window_size: stats |= sliding_stats - wandb.log({"actor/" + k: v for k, v in stats.items()}) + wandb.log({f"actor/{k}": v for k, v in stats.items()}) stats_writer.write(stats) self.init_stats() @@ -573,8 +573,9 @@ def run_actor_loop(cfg: DictConfig): data_stream = SingleStreamSpec(exp_path=exp_path, topic="actor") test_data_stream = SingleStreamSpec(exp_path=exp_path, topic="actor_test") - train_dataset = load_datasets(cfg.train_dataset_names) - test_dataset = load_datasets(cfg.test_dataset_names) + dataset_loader = hydra.utils.get_method(cfg.dataset_loader) + train_dataset = dataset_loader(cfg.train_dataset_names) + test_dataset = dataset_loader(cfg.test_dataset_names) if cfg.train_subset: train_dataset = train_dataset[cfg.train_subset.begin : cfg.train_subset.end] logger.info(f"Loaded {len(train_dataset)} training problems") @@ -611,7 +612,7 @@ def run_actor_loop(cfg: DictConfig): ] wait_for_inference_servers(llm_urls) - wait_for_verifier(cfg.verifier) + wait_for_environments(cfg) trainer_state = TrainerState(exp_path) if cfg.debug.mode in ["actor", "open_loop"]: trainer_state.propagated_weight_version = 0 diff --git a/pipelinerl/run_finetune.py b/pipelinerl/run_finetune.py index ade2073..74299a1 100644 --- a/pipelinerl/run_finetune.py +++ b/pipelinerl/run_finetune.py @@ -1,30 +1,30 @@ -from concurrent.futures import ThreadPoolExecutor -import logging - -import deepspeed -from accelerate.utils import FullyShardedDataParallelPlugin - import contextlib import json +import logging import os import threading import time from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict from functools import partial from pathlib import Path from queue import Empty, Queue -from typing import Any, List, Literal, Dict +from typing import Any, Dict, List, Literal +import deepspeed import requests import torch import torch.distributed as dist +from accelerate.utils import FullyShardedDataParallelPlugin +from omegaconf import DictConfig +from pydantic import BaseModel from torch.distributed.fsdp import FullStateDictConfig, StateDictType from torch.distributed.fsdp import FullyShardedDataParallel as FSDP from torch.distributed.fsdp.api import MixedPrecision +from transformers import PreTrainedTokenizerFast, get_scheduler, set_seed -from omegaconf import DictConfig -from pydantic import BaseModel +import pipelinerl.torch_utils from pipelinerl.finetune.checkpoints import ( load_model, load_tokenizer, @@ -37,23 +37,20 @@ from pipelinerl.finetune.data import collate, collate_packed from pipelinerl.finetune.logging_ import log_metrics, log_time, setup_logging from pipelinerl.finetune.optim import get_optimizer -from pipelinerl.finetune.utils import create_sentinel_batch, VersionedTensors from pipelinerl.finetune.rl import ( RLConfig, rl_step, ) from pipelinerl.finetune.rl.utils import get_avg_rl_stats from pipelinerl.finetune.types import TrainingMetrics -from transformers import get_scheduler, set_seed, PreTrainedTokenizerFast - -from pipelinerl.utils import wait_for_inference_servers -import pipelinerl.torch_utils +from pipelinerl.finetune.utils import VersionedTensors, create_sentinel_batch from pipelinerl.streams import ( SingleStreamSpec, read_stream, set_streams_backend, write_to_streams, ) +from pipelinerl.utils import wait_for_inference_servers logger = logging.getLogger(__name__) @@ -177,6 +174,7 @@ def run_dynamic_batch_size_data_loader( current_length = 0 samples_in_step = 0 sample_generator = sample_generator_fn(sample_queue) + skip_count = 0 while True: try: while True: @@ -185,9 +183,11 @@ def run_dynamic_batch_size_data_loader( sample_length = len(entry["input_ids"]) if entry else 0 if sample_length > max_seq_length: - raise ValueError( - f"Sample is of length {sample_length}, exceeding the max length of {max_seq_length}" + skip_count += 1 + logger.warning( + f"Sample length {sample_length} > max allowed length {max_seq_length}, skipping. Total {skip_count} samples skipped so far." ) + continue # check if adding current sample would exceed max_seq_length or if we've reached sample limit boundary = samples_in_step == samples_per_worker_per_step @@ -820,7 +820,7 @@ def toggle_sync(sync: bool): "stats/epoch": training_metrics.epoch, "stats/min_actor_version": lag_stats["min_version"], "stats/max_actor_version": lag_stats["max_version"], - "stats/queue_size": sample_queue.qsize(), + "stats/queue/samples": sample_queue.qsize(), "stats/time_waiting_for_data": training_metrics.time_waiting_for_data, "stats/lag": training_metrics.last_broadcasted_version - lag_stats["min_version"], "throughput/tokens_perGPU_per_sec": this_worker_tokens / sum(passes_took) if passes_took else 0, diff --git a/pipelinerl/run_preprocess.py b/pipelinerl/run_preprocess.py index fc03531..967e36c 100644 --- a/pipelinerl/run_preprocess.py +++ b/pipelinerl/run_preprocess.py @@ -2,42 +2,39 @@ os.environ["HF_DATASETS_DISABLE_PROGRESS_BARS"] = "1" -import multiprocessing as mp -from multiprocessing.managers import SharedMemoryManager -from concurrent.futures import ProcessPoolExecutor import logging +import multiprocessing as mp import queue -import time - -from litellm import BaseModel, Field - -from pipelinerl.utils import wait_for_inference_servers -from pipelinerl.world import WorldMap -from pipelinerl.finetune.logging_ import flatten_dict_config, init_wandb -from pipelinerl.shared_memory_array import SharedMemoryArray - -logger = logging.getLogger(__name__) import threading +import time +from concurrent.futures import ProcessPoolExecutor from functools import partial +from multiprocessing.managers import SharedMemoryManager from pathlib import Path from queue import Empty, Queue from typing import List import random -import transformers import datasets +import transformers +from litellm import BaseModel, Field + +from pipelinerl.finetune.logging_ import flatten_dict_config, init_wandb +from pipelinerl.shared_memory_array import SharedMemoryArray +from pipelinerl.utils import wait_for_inference_servers +from pipelinerl.world import WorldMap datasets.disable_caching() from datasets.arrow_dataset import Dataset from datasets.fingerprint import Hasher from omegaconf import DictConfig +from tapeagents.llms import TrainableLLM + from pipelinerl.finetune.checkpoints import ( load_tokenizer, ) from pipelinerl.finetune.data import preprocess_fn from pipelinerl.finetune.rl import RL_DATA_COLUMNS, RLConfig, populate_rl_data -from tapeagents.llms import TrainableLLM - from pipelinerl.streams import ( SingleStreamSpec, StreamRangeSpec, @@ -113,7 +110,7 @@ def replace_oov_tokens_with_the(data: list[dict], tokenizer: transformers.PreTra completion_start = len(entry["input_ids"]) - completion_length for i, token_id in enumerate(invalid_token_ids): if i + completion_start < len(entry["input_ids"]): - logger.warning(f"Invalid token in completion part, logprobs may be inconsistent") + logger.warning("Invalid token in completion part, logprobs may be inconsistent") entry["input_ids"] = new_input_ids new_data.append(entry) @@ -174,7 +171,7 @@ def run_dataset_loader( if len(buffer) == chunk_size: break if not _check_group_sizes(buffer, check_group_size): - raise ValueError(f"Invalid group sizes in data") + raise ValueError("Invalid group sizes in data") try: raw_chunk_queue.put_nowait(buffer) except queue.Full: @@ -294,7 +291,7 @@ def run_preprocessing_loop( partition_range=(0, max(world_map.total_finetune_gpus, 1)), ) stats_streams = SingleStreamSpec(exp_path=exp_root_dir, topic="preprocessor_stats") - logger.info(f"Streams initialized") + logger.info("Streams initialized") raw_chunk_queue = Queue(cfg.preprocess.queue_size) rl_config = RLConfig(**cfg.finetune.rl) @@ -405,8 +402,10 @@ def run_preprocessing_loop( stats = { "preprocessor/published_samples": published_samples, "preprocessor/published_model_version": max_model_version, - "preprocessor/samples_in_input_queue": raw_chunk_queue.qsize() * cfg.preprocess.chunk_size, - "preprocessor/samples_in_output_queue": samples_in_queue, + "preprocessor/queue/raw_samples": raw_chunk_queue.qsize() * cfg.preprocess.chunk_size, + "preprocessor/queue/raw": raw_chunk_queue.qsize(), + "preprocessor/queue/dataset_samples": samples_in_queue, + "preprocessor/queue/dataset": dataset_queue.qsize(), } if stats_aggregator.has_enough_data(): stats.update({"preprocessor/" + k: v for k, v in stats_aggregator.get_stats().items()}) diff --git a/pipelinerl/shared_memory_array.py b/pipelinerl/shared_memory_array.py index cc9b37f..b27cb4d 100644 --- a/pipelinerl/shared_memory_array.py +++ b/pipelinerl/shared_memory_array.py @@ -1,8 +1,8 @@ -import multiprocessing as mp -from multiprocessing.managers import SharedMemoryManager import pickle import struct -from typing import Any, Dict, List, Optional, Tuple, Union +from multiprocessing.managers import SharedMemoryManager +from typing import Any + class SharedMemoryArray: @@ -99,7 +99,7 @@ def __len__(self) -> int: """Return the number of entries in the array.""" return self.num_entries - def clear(self, index: int = None) -> None: + def clear(self, index: int | None = None) -> None: """ Clear an entry or the entire array. diff --git a/pipelinerl/tapeagents_rollouts.py b/pipelinerl/tapeagents_rollouts.py new file mode 100644 index 0000000..f26429d --- /dev/null +++ b/pipelinerl/tapeagents_rollouts.py @@ -0,0 +1,68 @@ +import asyncio +import time + +import aiohttp +from omegaconf import DictConfig +from tapeagents.agent import Agent, LLMEvent, LLMStream +from tapeagents.core import Prompt, StopStep, TrainingText +from tapeagents.dialog_tape import DialogTape, UserStep +from tapeagents.environment import Environment +from tapeagents.llms.trainable import TrainableLLM +from tapeagents.orchestrator import get_agent_and_env_from_config, main_loop + +from pipelinerl.async_llm import llm_async_generate +from pipelinerl.math.rollouts import RolloutResult + + +def run_tapeagent( + task: str, agent: Agent, environment: Environment, max_loops: int +) -> tuple[list[TrainingText], dict[str, float]]: + start_tape = DialogTape(steps=[UserStep(content=task)]) + tape: DialogTape | None = None + for event in main_loop(agent, start_tape, environment, max_loops): + if event.agent_tape: + tape = event.agent_tape + elif event.env_tape: + tape = event.env_tape + assert tape is not None, "No tape generated" + has_errors = any([1 for s in tape.steps if s.llm_dict().get("error")]) + has_answer = any([isinstance(s, StopStep) for s in tape.steps]) + _, llm_calls = agent.reuse(tape) + samples = [agent.make_training_text(llm_call) for llm_call in llm_calls] + reward = 0 # TODO: implement verifier usage and reward calculation + metrics = { + "reward": reward, + "success": reward > 0, + "no_error": not has_errors, + "no_answer": not has_answer, + "prompt_tokens": sum([llm_call.prompt_length_tokens for llm_call in llm_calls]), + "output_tokens": sum([llm_call.output_length_tokens for llm_call in llm_calls]), + "overflow": 0, # TODO: should we treat max_loops stop as overflow? + } + return samples, metrics + + +async def generate_rollout( + cfg: DictConfig, + llm: TrainableLLM, + problem: dict, + session: aiohttp.ClientSession, +) -> RolloutResult: + def generate(self, prompt: Prompt): + # !! should be called in a separate thread only !! + # 'self' here is the llm instance (agent.llms["default"]) + # 'session' is captured from the outer scope of generate_rollout + def _implementation(): + llm_call = asyncio.run(llm_async_generate(self, prompt, session)) + yield LLMEvent(output=llm_call.output, llm_call=llm_call) + + return LLMStream(_implementation(), prompt) + + time_start = time.time() + task: str = cfg.task_template.format(task=problem["task"]) + agent, environment = get_agent_and_env_from_config(cfg) + agent.llms = {"default": llm.model_copy()} + agent.llms["default"].generate = generate # type: ignore + samples, metrics = await asyncio.to_thread(run_tapeagent, task, agent, environment, cfg.max_loops) + latency = time.time() - time_start + return RolloutResult(training_texts=samples, metrics=metrics, latency=latency, dataset_name=problem.get("dataset")) diff --git a/pipelinerl/utils.py b/pipelinerl/utils.py index 5d634a4..42592c1 100644 --- a/pipelinerl/utils.py +++ b/pipelinerl/utils.py @@ -10,6 +10,7 @@ from typing import Dict, Mapping, Optional, TextIO, Union, List import threading import numpy as np +from omegaconf import DictConfig import psutil import requests import torch @@ -17,6 +18,7 @@ from transformers import PreTrainedTokenizer from collections import defaultdict +from pipelinerl.world import Job from tapeagents.llms import LLMOutput from tapeagents.core import Prompt @@ -238,6 +240,24 @@ def wait_for_inference_servers(urls: list[str]): logger.info("All inference servers are up") +def wait_for_environments(cfg: DictConfig): + """ + Wait for the verifier to be ready. + """ + env_jobs = [Job(**job) for job in cfg.jobs if job.kind == "environment"] + for job in env_jobs: + while True: + url = f"http://{job.hostname}:{job.port}/health" + # use requests + try: + response = requests.get(url) + if response.status_code == 200: + break + except: + logger.info(f"Waiting for environment at {url} to be ready...") + time.sleep(5.0) + + @contextlib.contextmanager def better_crashing(entrypoint_name: str): try: diff --git a/pipelinerl/world.py b/pipelinerl/world.py index 1f3ebf7..1c74afb 100644 --- a/pipelinerl/world.py +++ b/pipelinerl/world.py @@ -1,5 +1,6 @@ import logging import os +from typing import Literal from pydantic import BaseModel from omegaconf import DictConfig import torch @@ -9,14 +10,18 @@ class Job(BaseModel): """Represent the decision to launch a replica of a particular worker (e.g. actor) at a particular rank""" - + # The job kind kind: str - # The global index of this job among jobs of the same kind + # The global index of this job among all jobs + idx: int + # The index of this job among jobs of the same kind replica_idx: int # The index of this job among similar jobs on the same node local_idx: int = 0 # Where this job should run node_rank: int + hostname: str + port: int | None = None # Which GPUs the job will use gpus: list[int] = [] # The URL of the job @@ -58,17 +63,22 @@ def __init__(self, cfg: DictConfig, verbose: bool = False): self.weight_update_group_size = 1 # Place jobs on nodes in a reverse order to make sure that last node has a finetuning job going on - self.available_gpus: dict[int, set] = {i: set(range(self.node_size)) for i in reversed(range(self.world_size))} + self.available_gpus = {i: set(range(self.node_size)) for i in reversed(range(self.world_size))} + self.cpu_heavy_jobs = {i: 0 for i in range(self.world_size)} self.job_map = {i: [] for i in range(self.world_size)} + self.total_jobs = 0 if place_inference_jobs: self._place_inference_jobs(cfg) + self._place_pipeline_stages(cfg) + if cfg.environment: + self._place_environments(cfg) # Place the finetune workers on the remaining gpus, take all remaining GPUs for node, remaining_gpus in self.available_gpus.items(): gpus = list(remaining_gpus) if gpus: - self.job_map[node].append(Job(kind="finetune", replica_idx=node, node_rank=node, gpus=gpus)) + self.add_job(node_rank=node, kind="finetune", replica_idx=node, gpus=gpus) # Pretty-log the world map self._log_info("--- WORLD MAP ---") @@ -77,6 +87,28 @@ def __init__(self, cfg: DictConfig, verbose: bool = False): for job in jobs: self._log_info(f" {job.kind} {job.replica_idx} on gpus {job.gpus}, local idx {job.local_idx}") + def add_job(self, node_rank: int, kind: str, replica_idx: int, local_idx: int = 0, port: int | None = None, gpus: list[int] | None = None, cpu_heavy: bool = False, url: str = "") -> Job: + """Add a job to the world map.""" + if gpus is None: + gpus = [] + job = Job( + kind=kind, + idx=self.total_jobs, + replica_idx=replica_idx, + local_idx=local_idx, + node_rank=node_rank, + hostname=self.address_map[node_rank], + port=port, + gpus=gpus, + url=url + ) + self.job_map[node_rank].append(job) + self.total_jobs += 1 + if cpu_heavy: + self.cpu_heavy_jobs[node_rank] += 1 + return job + + def _split_gpus_by_purpose(self, cfg): fraction_sum = cfg.world.actor_fraction + cfg.world.preprocessor_fraction + cfg.world.finetune_fraction actor_fraction = cfg.world.actor_fraction / fraction_sum @@ -94,27 +126,27 @@ def _split_gpus_by_purpose(self, cfg): f"{desired_preprocessor_gpu_share} for preprocessors, {desired_finetune_gpu_share} for finetune" ) - gpus_per_actor = int(desired_actor_gpu_share / cfg.world.actors) if cfg.world.actors > 0 else 0 + gpus_per_actor = int(desired_actor_gpu_share / cfg.world.replicas) if cfg.world.replicas > 0 else 0 gpus_per_actor = gpus_per_actor - (gpus_per_actor % self.gpus_per_llm) gpus_per_preprocessor = ( - int(desired_preprocessor_gpu_share / cfg.world.preprocessors) if cfg.world.preprocessors > 0 else 0 + int(desired_preprocessor_gpu_share / cfg.world.replicas) if cfg.world.replicas > 0 else 0 ) gpus_per_preprocessor = gpus_per_preprocessor - (gpus_per_preprocessor % self.gpus_per_llm) self.llms_per_actor = max(int(gpus_per_actor / self.gpus_per_llm), 1) if gpus_per_actor > 0 else 0 - self.total_actor_llms = self.llms_per_actor * cfg.world.actors + self.total_actor_llms = self.llms_per_actor * cfg.world.replicas self.llms_per_preprocessor = ( max(int(gpus_per_preprocessor / self.gpus_per_llm), 1) if gpus_per_preprocessor > 0 else 0 ) self.gpus_per_actor = gpus_per_actor self.gpus_per_preprocessor = gpus_per_preprocessor - total_actor_gpus = cfg.world.actors * gpus_per_actor - total_preprocessor_gpus = cfg.world.preprocessors * gpus_per_preprocessor + total_actor_gpus = cfg.world.replicas * gpus_per_actor + total_preprocessor_gpus = cfg.world.replicas * gpus_per_preprocessor self.total_finetune_gpus = total_gpus - total_actor_gpus - total_preprocessor_gpus self._log_info( f"The configuration required:\n" f"{desired_actor_gpu_share} for actors, {desired_preprocessor_gpu_share} for preprocessors, {self.total_finetune_gpus} for finetune,\n" - f"with {cfg.world.actors} actors and {cfg.world.preprocessors} preprocessors,\n" + f"with {cfg.world.replicas} actors and {cfg.world.replicas} preprocessors,\n" f"and with {self.gpus_per_llm} per each LLM.\n" ) self._log_info("I have adjusted the GPU shares to accomodate these constraints.") @@ -128,62 +160,72 @@ def _split_gpus_by_purpose(self, cfg): self.weight_update_group_size = self.total_actor_llms * self.gpus_per_llm + 1 + def _place_pipeline_stages(self, cfg): + for worker_idx in range(cfg.world.replicas): + node = self.get_least_busy_node() + self.add_job(kind="actor", replica_idx=worker_idx, node_rank=node, gpus=[], cpu_heavy=True) + self.add_job(kind="preprocessor", replica_idx=worker_idx, node_rank=node, gpus=[], cpu_heavy=True) + + def _place_environments(self, cfg): + for worker_idx in range(cfg.world.env_replicas): + node = self.get_least_busy_node() + envs_at_node = len([job for job in self.job_map[node] if job.kind == "environment"]) + self.add_job( + kind="environment", + replica_idx=worker_idx, + node_rank=node, + port=cfg.world.environment_start_port + envs_at_node, + gpus=[], + cpu_heavy=True, + ) + def _place_inference_jobs(self, cfg): - actor_placed = False - for worker_idx in range(cfg.world.actors): + for _ in range(cfg.world.replicas): for actor_llm_idx in range(self.llms_per_actor): node = next( (node for node in self.available_gpus if len(self.available_gpus[node]) >= self.gpus_per_llm), None ) if node is None: raise ValueError("Not enough gpus to place all actors") - if not actor_placed: - self.job_map[node].append(Job(kind="actor", replica_idx=worker_idx, node_rank=node, gpus=[])) - self.job_map[node].append(Job(kind="verifier", replica_idx=worker_idx, node_rank=node, gpus=[])) - actor_placed = True gpus = [self.available_gpus[node].pop() for _ in range(self.gpus_per_llm)] local_idx = min(gpus) llm_url = f"http://{self.address_map[node]}:{8080 + local_idx}" - self.job_map[node].append( - Job( - kind="actor_llm", - replica_idx=actor_llm_idx, - local_idx=local_idx, - node_rank=node, - gpus=gpus, - url=llm_url, - ) + self.add_job( + kind="actor_llm", + replica_idx=actor_llm_idx, + local_idx=local_idx, + node_rank=node, + gpus=gpus, + port=8080 + local_idx, + url=llm_url, ) - preprocessor_placed = False - for worker_idx in range(cfg.world.preprocessors): + for _ in range(cfg.world.replicas): for preprocessor_llm_idx in range(self.llms_per_preprocessor): node = next( (node for node in self.available_gpus if len(self.available_gpus[node]) >= self.gpus_per_llm), None ) if node is None: raise ValueError("Not enough gpus to place all preprocessors") - if not preprocessor_placed: - self.job_map[node].append(Job(kind="preprocessor", replica_idx=worker_idx, node_rank=node, gpus=[])) - preprocessor_placed = True gpus = [self.available_gpus[node].pop() for _ in range(self.gpus_per_llm)] local_idx = min(gpus) ref_url = f"http://{self.address_map[node]}:{8180 + local_idx}" - self.job_map[node].append( - Job( - kind="preprocessor_llm", - replica_idx=preprocessor_llm_idx, - local_idx=local_idx, - node_rank=node, - gpus=gpus, - url=ref_url, - ) + self.add_job( + kind="preprocessor_llm", + replica_idx=preprocessor_llm_idx, + local_idx=local_idx, + node_rank=node, + gpus=gpus, + url=ref_url, ) - if not preprocessor_placed: - assert cfg.world.preprocessor_fraction == 0 - self.job_map[self.world_size - 1].append( - Job(kind="preprocessor", replica_idx=0, node_rank=self.world_size - 1, gpus=[]) - ) + + def get_least_busy_node(self): + """Get the node with the least number of CPU-heavy jobs.""" + result = 0 + for node, cpu_heavy_jobs in self.cpu_heavy_jobs.items(): + if cpu_heavy_jobs < self.cpu_heavy_jobs[result]: + result = node + return result def my_jobs(self) -> list[Job]: return self.job_map[self.my_rank] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4f10035 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pipelinerl" +version = "0.1.0" +description = "A scalable asynchronous reinforcement learning implementation with in-flight weight updates." +readme = "README.md" +requires-python = ">=3.11" +license = { file = "LICENSE" } +authors = [ + { name = "ServiceNow" }, +] +dependencies = [ + "torch>=2.6", + "vllm==0.8.3", + "accelerate @ git+https://github.com/ServiceNow/accelerate.git@v1.2.0-hotfix", + "Tapeagents[finetune]==0.1.12", + "transformers==4.51.0", + "flash-attn==2.7.4.post1", + "math-verify[antlr4_9_3]==0.7.0", + "orjson==3.10.16", + "redis==5.2.1", + "hydra-core>=1.3.2", +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["pipelinerl*"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f67cf13..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -vllm==0.8.3 -accelerate @ git+https://github.com/ServiceNow/accelerate.git@v1.2.0-hotfix -Tapeagents[finetune]==0.1.8 -transformers==4.51.0 -flash-attn==2.7.4.post1 -math-verify[antlr4_9_3]==0.7.0 -orjson==3.10.16 -redis==5.2.1 \ No newline at end of file